001    /* 
002     * Copyright 2008-2009 the original author or authors.
003     * The contents of this file are subject to the Mozilla Public License
004     * Version 1.1 (the "License"); you may not use this file except in
005     * compliance with the License. You may obtain a copy of the License at
006     * http://www.mozilla.org/MPL/
007     *
008     * Software distributed under the License is distributed on an "AS IS"
009     * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the
010     * License for the specific language governing rights and limitations
011     * under the License.
012     */
013     
014    package com.mtgi.analytics;
015    
016    import static java.util.UUID.randomUUID;
017    
018    import java.io.BufferedOutputStream;
019    import java.io.File;
020    import java.io.FileFilter;
021    import java.io.FileNotFoundException;
022    import java.io.FileOutputStream;
023    import java.io.IOException;
024    import java.io.OutputStream;
025    import java.text.SimpleDateFormat;
026    import java.util.Arrays;
027    import java.util.Comparator;
028    import java.util.Date;
029    import java.util.Queue;
030    import java.util.regex.Matcher;
031    import java.util.regex.Pattern;
032    import java.util.zip.GZIPOutputStream;
033    
034    import javax.xml.stream.XMLOutputFactory;
035    import javax.xml.stream.XMLStreamException;
036    import javax.xml.stream.XMLStreamWriter;
037    
038    import org.apache.commons.logging.Log;
039    import org.apache.commons.logging.LogFactory;
040    import org.springframework.beans.factory.InitializingBean;
041    import org.springframework.beans.factory.DisposableBean;
042    import org.springframework.beans.factory.annotation.Required;
043    import org.springframework.jmx.export.annotation.ManagedAttribute;
044    import org.springframework.jmx.export.annotation.ManagedOperation;
045    import org.springframework.jmx.export.annotation.ManagedOperationParameter;
046    import org.springframework.jmx.export.annotation.ManagedResource;
047    
048    import com.mtgi.io.RelocatableFile;
049    import com.sun.xml.fastinfoset.stax.StAXDocumentSerializer;
050    
051    /**
052     * Behavior Tracking persister which writes events to an XML log file,
053     * either as plain text or FastInfoset binary XML.  Which format is
054     * selected by {@link #setBinary(boolean)}.  Log rotation can be accomplished
055     * by {@link #rotateLog()}.
056     */
057    @ManagedResource(objectName="com.mtgi:group=analytics,name=BehaviorTrackingLog", 
058                                     description="Perform maintenance on BehaviorTracking XML logfiles")
059    public class XmlBehaviorEventPersisterImpl 
060            implements BehaviorEventPersister, InitializingBean, DisposableBean 
061    {
062            private static final Log log = LogFactory.getLog(XmlBehaviorEventPersisterImpl.class);
063    
064            private static final SimpleDateFormat DEFAULT_DATE_FORMAT = new SimpleDateFormat("yyyyMMddHHmmss");
065            private static final Pattern FILE_NAME_PATTERN = Pattern.compile("^(.+)\\.b?xml(?:\\.gz)?");
066            
067            private boolean binary;
068            private boolean compress;
069            private File file;
070            private SimpleDateFormat dateFormat = DEFAULT_DATE_FORMAT;
071    
072            private XMLStreamWriter writer;
073            private OutputStream stream;
074            
075            /** Set to true to log in FastInfoset binary XML format.  Defaults to false. */
076            @ManagedAttribute(description="Can be used to switch between binary and text XML.  Changes take affect after the next log rotation.")
077            public void setBinary(boolean binary) {
078                    this.binary = binary;
079            }
080            @ManagedAttribute(description="Can be used to switch between binary and text XML.  Changes take affect after the next log rotation.")
081            public boolean isBinary() {
082                    return binary;
083            }
084    
085            @ManagedAttribute(description="Can be used to turn on/off log file compression.  Changes take affect after the next log rotation.")
086            public boolean isCompress() {
087                    return compress;
088            }
089            /** Set to true to log in ZLIB compressed format.  Changes take affect after the next log rotation.  Defaults to false. */
090            @ManagedAttribute(description="Can be used to turn on/off log file compression.  Changes take affect after the next log rotation.")
091            public void setCompress(boolean compress) {
092                    this.compress = compress;
093            }
094            
095            /** override the default log name date format */
096            public void setDateFormat(String dateFormat) {
097                    this.dateFormat = new SimpleDateFormat(dateFormat);
098            }
099    
100            /**
101             * JMX operation to list archived xml data files available for download.
102             */
103            @ManagedOperation(description="List all archived performance data log files available for download")
104            public StringBuffer listLogFiles() {
105    
106                    final Pattern pattern = getArchiveNamePattern();
107    
108                    //scan parent directory for matches.
109                    File[] files = file.getParentFile().listFiles(new FileFilter() {
110                            public boolean accept(File child) {
111                                    String childName = child.getName();
112                                    return pattern.matcher(childName).matches();
113                            }
114                    });
115                    Arrays.sort(files, FileOrder.INST);
116    
117                    StringBuffer ret = new StringBuffer("<html><body><table><tr><th>File</th><th>Size</th><th>Modified</th></tr>");
118                    for (File f : files)
119                            ret.append("<tr><td>").append(f.getName()).append("</td><td>")
120                               .append(f.length()).append("</td><td>")
121                               .append(new Date(f.lastModified()).toString()).append("</td></tr>");
122                    ret.append("</table></body></html>");
123                    return ret;
124            }
125            
126            @ManagedOperation(description="directly access a performance log file by name; use listLogFiles to determine valid file names")
127            @ManagedOperationParameter(name="name", description="The relative name of the file to download; see listLogFiles")
128            public RelocatableFile downloadLogFile(String name) throws IOException {
129                    if (!getArchiveNamePattern().matcher(name).matches())
130                            throw new IllegalArgumentException(name + " does not appear to be a performance data file");
131                    File data = new File(file.getParentFile(), name);
132                    if (!data.isFile())
133                            throw new FileNotFoundException(file.getAbsolutePath());
134                    return new RelocatableFile(data);
135            }
136            
137            /** set the destination log file.  The file extension will be modified based on compression / binary settings. */
138            @Required
139            public void setFile(String path) {
140                    this.file = new File(path);
141            }
142            
143            public String getFile() {
144                    return file == null ? null : file.getAbsolutePath();
145            }
146            
147            public void afterPropertiesSet() throws Exception {
148                    File dir = file.getCanonicalFile().getParentFile();
149                    if (!dir.isDirectory() && !dir.mkdirs())
150                            throw new IOException("Unable to create directories for log file " + file.getAbsolutePath());
151                    
152                    //open for business.
153                    rotateLog();
154            }
155    
156            public void destroy() throws Exception {
157                    closeWriter();
158            }
159            
160            @ManagedAttribute(description="Report the current size of the XML log, in bytes")
161            public long getFileSize() {
162                    return file.length();
163            }
164    
165            public int persist(Queue<BehaviorEvent> events) {
166                    int count = 0;
167                    try {
168                            BehaviorEventSerializer serializer = new BehaviorEventSerializer();
169                            
170                            while (!events.isEmpty()) {
171                                    BehaviorEvent event = events.remove();
172                                    if (event.getId() == null)
173                                            event.setId(randomUUID());
174                                    
175                                    BehaviorEvent parent = event.getParent();
176                                    if (parent != null && parent.getId() == null)
177                                            parent.setId(randomUUID());
178                                    
179                                    synchronized (this) {
180                                            serializer.serialize(writer, event);
181                                            writer.writeCharacters("\n");
182                                    }
183                                    
184                                    ++count;
185                            }
186                            
187                            synchronized (this) {
188                                    writer.flush();
189                                    stream.flush();
190                            }
191                            
192                    } catch (Exception error) {
193                            log.error("Error persisting events; discarding " + events.size() + " events without saving", error);
194                            events.clear();
195                    }
196                    
197                    return count;
198            }
199    
200            /**
201             * Force a rotation of the log.  The new archive log will be named <code>[logfile].yyyyMMddHHmmss</code>.
202             * If a file of that name already exists, an extra _N will be appended to it, where N is an
203             * integer sufficiently large to make the name unique.  This method can be called
204             * externally by the Quartz scheduler for periodic rotation, or by an administrator
205             * via JMX.
206             */
207            @ManagedOperation(description="Force a rotation of the behavior tracking log")
208            public String rotateLog() throws IOException, XMLStreamException {
209                    StringBuffer msg = new StringBuffer();
210                    synchronized (this) {
211                            //flush current writer and close streams.
212                            closeWriter();
213                             
214                            //rotate old log data to timestamped archive file.
215                            File archived = moveToArchive();
216                            if (archived == null)
217                                    msg.append("No existing log data.");
218                            else
219                                    msg.append(archived.getAbsolutePath());
220                                    
221                            //update output file name, in case binary/compress settings changed since last rotate.
222                            file = getLogFile(file);
223                            
224                            //perform archive again, in case our file name changed and has pointed us at a preexisting file;
225                            //this can happen if the system wasn't shut down cleanly the last time.
226                            moveToArchive();
227    
228                            //open a new stream, optionally compressed.
229                            if (compress)
230                                    stream = new GZIPOutputStream(new FileOutputStream(file));
231                            else
232                                    stream = new BufferedOutputStream(new FileOutputStream(file));
233                            
234                            //open a new writer over the stream.
235                            if (binary) {
236                                    StAXDocumentSerializer sds = new StAXDocumentSerializer();
237                                    sds.setOutputStream(stream);
238                                    writer = sds;
239                                    writer.writeStartDocument();
240                                    writer.writeStartElement("event-log");
241                            } else {
242                                    writer = XMLOutputFactory.newInstance().createXMLStreamWriter(stream);
243                            }
244                    }
245                    return msg.toString();
246            }
247            
248            /** 
249             * if the current target log location exists, move it to archive location, returning the newly created archive file 
250             * @return the archive file; or null if the current target location does not yet exist
251             * @throws IOException if an archive file was needed, but could not by created
252             */
253            private File moveToArchive() throws IOException {
254                    if (file.exists()) {
255                            File archive = getArchiveFile();
256                            if (!file.renameTo(archive))
257                                    throw new IOException("Unable to rename log to " + archive.getAbsolutePath() + "; is " + file.getPath() + " open by another process?");
258                            return archive;
259                    }
260                    return null;
261            }
262            
263            private void closeWriter() {
264                    if (writer != null) {
265                            //finish writing XML document.
266                            try {
267                                    if (binary)
268                                            writer.writeEndDocument();
269                                    writer.flush();
270                            } catch (XMLStreamException xse) {
271                                    log.error("Error flushing log for rotation", xse);
272                            } finally {
273                                    
274                                    //finish the compressed stream, if applicable.
275                                    try {
276                                            if (stream instanceof GZIPOutputStream)
277                                                    ((GZIPOutputStream)stream).finish();
278                                    } catch (IOException ioe) {
279                                            log.error("Error finishing zip stream", ioe);
280                                    } finally {
281    
282                                            //close the file
283                                            try {
284                                                    stream.close();
285                                            } catch (IOException ioe) {
286                                                    log.error("Error closing log stream", ioe);
287                                            } finally {
288                                                    writer = null;
289                                                    stream = null;
290                                            }
291                                            
292                                    }
293                            }
294                    }
295            }
296            
297            /** get the active log file, modifying the supplied base name to reflect compressed / binary options. */
298            public File getLogFile(File file) {
299    
300                    //generate an extension based on output options.
301                    String ext = isBinary() ? ".bxml" : ".xml";
302                    if (isCompress())
303                            ext += ".gz";
304                    
305                    //strip off current extension if applicable.
306                    String baseName = file.getName();
307                    Matcher matcher = FILE_NAME_PATTERN.matcher(baseName);
308                    if (matcher.matches())
309                            baseName = matcher.group(1);
310                    
311                    return new File(file.getParentFile(), baseName + ext);
312            }
313            
314            /** get the archive file to which current log data should be moved on rotation */
315            private File getArchiveFile() {
316                    String baseName = file.getPath() + "." + dateFormat.format(new Date());
317                    File ret = new File(baseName);
318                    for (int i = 1; ret.exists() && i < 11; ++i)
319                            ret = new File(baseName + "_" + i);
320                    if (ret.exists())
321                            throw new IllegalStateException("Unable to create a unique file name starting with " + baseName + "; maybe system date is off?");
322                    return ret;
323            }
324            
325            private Pattern getArchiveNamePattern() {
326                    //build a regex that matches files with the configured basename, plus a non-binary or binary xml extension,
327                    //plus a date suffix.
328                    String filePattern = file.getName();
329                    Matcher m = FILE_NAME_PATTERN.matcher(filePattern);
330                    if (m.matches())
331                            filePattern = m.group(1);
332                    filePattern += "\\.b?xml\\..+";
333                    return Pattern.compile(filePattern);
334            }
335            
336            protected static class FileOrder implements Comparator<File> {
337    
338                    public static final FileOrder INST = new FileOrder();
339                    
340                    public int compare(File f0, File f1) {
341                            long delta = f0.lastModified() - f1.lastModified();
342                            return delta == 0 ? f0.compareTo(f1)
343                                                              : delta > 0 ? -1
344                                                              : 0;
345                    }
346                    
347            }
348    }