001package com.logfire.logback;
002
003import ch.qos.logback.classic.encoder.PatternLayoutEncoder;
004import ch.qos.logback.classic.spi.ILoggingEvent;
005import ch.qos.logback.classic.spi.IThrowableProxy;
006import ch.qos.logback.core.UnsynchronizedAppenderBase;
007import com.fasterxml.jackson.annotation.JsonInclude;
008import com.fasterxml.jackson.core.JsonProcessingException;
009import com.fasterxml.jackson.databind.Module;
010import com.fasterxml.jackson.databind.ObjectMapper;
011import com.fasterxml.jackson.databind.PropertyNamingStrategies;
012import org.slf4j.Logger;
013import org.slf4j.LoggerFactory;
014
015import java.io.IOException;
016import java.io.OutputStream;
017import java.net.HttpURLConnection;
018import java.net.URL;
019import java.nio.charset.StandardCharsets;
020import java.util.*;
021import java.util.Map.Entry;
022import java.util.concurrent.atomic.AtomicBoolean;
023import java.util.concurrent.Executors;
024import java.util.concurrent.ScheduledExecutorService;
025import java.util.concurrent.ScheduledFuture;
026import java.util.concurrent.ThreadFactory;
027import java.util.concurrent.TimeUnit;
028import java.util.stream.Collectors;
029
030public class LogfireAppender extends UnsynchronizedAppenderBase<ILoggingEvent> {
031
032    // Customizable variables
033    protected String appName;
034    protected String ingestUrl = "https://in.logfire.ai";
035
036    protected String sourceToken;
037    protected String userAgent = "Logfire Logback Appender";
038
039    protected List<String> mdcFields = new ArrayList<>();
040    protected List<String> mdcTypes = new ArrayList<>();
041
042    protected int maxQueueSize = 100000;
043    protected int batchSize = 1000;
044    protected int batchInterval = 3000;
045    protected int connectTimeout = 5000;
046    protected int readTimeout = 10000;
047    protected int maxRetries = 5;
048    protected int retrySleepMilliseconds = 300;
049
050    protected PatternLayoutEncoder encoder;
051
052    // Non-customizable variables
053    protected Vector<ILoggingEvent> batch = new Vector<>();
054    protected AtomicBoolean isFlushing = new AtomicBoolean(false);
055    protected boolean mustReflush = false;
056    protected boolean warnAboutMaxQueueSize = true;
057
058    // Utils
059    protected ScheduledExecutorService scheduledExecutorService;
060    protected ScheduledFuture<?> scheduledFuture;
061    protected ObjectMapper dataMapper;
062    protected Logger logger;
063    protected int retrySize = 0;
064    protected int retries = 0;
065    protected boolean disabled = false;
066
067    protected ThreadFactory threadFactory = r -> {
068        Thread thread = Executors.defaultThreadFactory().newThread(r);
069        thread.setName("logfire-appender");
070        thread.setDaemon(true);
071        return thread;
072    };
073
074    public LogfireAppender() {
075        logger = LoggerFactory.getLogger(LogfireAppender.class);
076
077        dataMapper = new ObjectMapper()
078                .setSerializationInclusion(JsonInclude.Include.NON_NULL)
079                .setPropertyNamingStrategy(PropertyNamingStrategies.UPPER_CAMEL_CASE);
080
081        scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(threadFactory);
082        scheduledFuture = scheduledExecutorService.scheduleWithFixedDelay(new LogfireSender(), batchInterval, batchInterval, TimeUnit.MILLISECONDS);
083    }
084
085    @Override
086    protected void append(ILoggingEvent event) {
087        if (disabled)
088            return;
089
090        if (event.getLoggerName().equals(LogfireAppender.class.getName()))
091            return;
092
093        if (this.ingestUrl.isEmpty() || this.sourceToken == null || this.sourceToken.isEmpty()) {
094            // Prevent potential deadlock, when a blocking logger is configured - avoid using logger directly in append
095            startThread("logfire-warning-logger", () -> {
096                logger.warn("Missing Source token for Logfire - disabling LogfireAppender. Find out how to fix this at: https://logfire.ai/docs/logs/java ");
097            });
098            this.disabled = true;
099            return;
100        }
101
102        if (batch.size() < maxQueueSize) {
103            batch.add(event);
104        }
105
106        if (warnAboutMaxQueueSize && batch.size() == maxQueueSize) {
107            this.warnAboutMaxQueueSize = false;
108            // Prevent potential deadlock, when a blocking logger is configured - avoid using logger directly in append
109            startThread("logfire-error-logger", () -> {
110                logger.error("Maximum number of messages in queue reached ({}). New messages will be dropped.", maxQueueSize);
111            });
112        }
113
114        if (batch.size() >= batchSize) {
115            if (isFlushing.get())
116                return;
117
118            startThread("logfire-appender-flush", new LogfireSender());
119        }
120    }
121
122    protected void startThread(String threadName, Runnable runnable) {
123        Thread thread = Executors.defaultThreadFactory().newThread(runnable);
124        thread.setName(threadName);
125        thread.start();
126    }
127
128    protected void flush() {
129        if (batch.isEmpty())
130            return;
131
132        // Guaranteed to not be running concurrently
133        if (isFlushing.getAndSet(true))
134            return;
135
136        mustReflush = false;
137
138        int flushedSize = batch.size();
139        if (flushedSize > batchSize) {
140            flushedSize = batchSize;
141            mustReflush = true;
142        }
143        if (retries > 0 && flushedSize > retrySize) {
144            flushedSize = retrySize;
145            mustReflush = true;
146        }
147
148        if (!flushLogs(flushedSize)) {
149            mustReflush = true;
150        }
151
152        isFlushing.set(false);
153
154        if (mustReflush || batch.size() >= batchSize) {
155            flush();
156        }
157    }
158
159    protected boolean flushLogs(int flushedSize) {
160        retrySize = flushedSize;
161
162        try {
163            if (retries > maxRetries) {
164                batch.subList(0, flushedSize).clear();
165                logger.error("Dropped batch of {} logs.", flushedSize);
166                warnAboutMaxQueueSize = true;
167                retries = 0;
168
169                return true;
170            }
171
172            if (retries > 0) {
173                logger.info("Retrying to send {} logs to Logfire ({} / {})", flushedSize, retries, maxRetries);
174                try {
175                    TimeUnit.MILLISECONDS.sleep(retrySleepMilliseconds);
176                } catch (InterruptedException e) {
177                    // Continue
178                }
179            }
180
181            LogfireResponse response = callHttpURLConnection(flushedSize);
182
183            if (response.getStatus() >= 300 || response.getStatus() < 200) {
184                logger.error("Error calling Logfire : {} ({})", response.getError(), response.getStatus());
185                retries++;
186
187                return false;
188            }
189
190            batch.subList(0, flushedSize).clear();
191            warnAboutMaxQueueSize = true;
192            retries = 0;
193
194            return true;
195
196        } catch (ConcurrentModificationException e) {
197            logger.error("Error clearing {} logs from batch, will retry immediately.", flushedSize, e);
198            retries = maxRetries; // No point in retrying to send the data
199
200        } catch (JsonProcessingException e) {
201            logger.error("Error processing JSON data : {}", e.getMessage(), e);
202            retries = maxRetries; // No point in retrying when batch cannot be processed into JSON
203
204        } catch (Exception e) {
205            logger.error("Error trying to call Logfire : {}", e.getMessage(), e);
206        }
207
208        retries++;
209
210        return false;
211    }
212
213    protected LogfireResponse callHttpURLConnection(int flushedSize) throws IOException {
214        HttpURLConnection connection = getHttpURLConnection();
215
216        try {
217            connection.connect();
218        } catch (Exception e) {
219            logger.error("Error trying to call Logfire : {}", e.getMessage(), e);
220        }
221
222        try (OutputStream os = connection.getOutputStream()) {
223            byte[] input = batchToJson(flushedSize).getBytes(StandardCharsets.UTF_8);
224            os.write(input, 0, input.length);
225            os.flush();
226        }
227
228        connection.disconnect();
229
230        return new LogfireResponse(connection.getResponseMessage(), connection.getResponseCode());
231    }
232
233    protected HttpURLConnection getHttpURLConnection() throws IOException {
234        HttpURLConnection httpURLConnection = (HttpURLConnection) new URL(this.ingestUrl).openConnection();
235        httpURLConnection.setDoOutput(true);
236        httpURLConnection.setDoInput(true);
237        httpURLConnection.setRequestProperty("User-Agent", this.userAgent);
238        httpURLConnection.setRequestProperty("Accept", "application/json");
239        httpURLConnection.setRequestProperty("Content-Type", "application/json");
240        httpURLConnection.setRequestProperty("Charset", "UTF-8");
241        httpURLConnection.setRequestProperty("Authorization", String.format("Bearer %s", this.sourceToken));
242        httpURLConnection.setRequestMethod("POST");
243        httpURLConnection.setConnectTimeout(this.connectTimeout);
244        httpURLConnection.setReadTimeout(this.readTimeout);
245        return httpURLConnection;
246    }
247
248    protected String batchToJson(int flushedSize) throws JsonProcessingException {
249        return this.dataMapper.writeValueAsString(
250            new ArrayList<>(batch.subList(0, flushedSize))
251                .stream()
252                .map(this::buildPostData)
253                .collect(Collectors.toList())
254        );
255    }
256
257    protected Map<String, Object> buildPostData(ILoggingEvent event) {
258        Map<String, Object> logLine = new HashMap<>();
259        logLine.put("dt", Long.toString(event.getTimeStamp()));
260        logLine.put("level", event.getLevel().toString());
261        logLine.put("app", this.appName);
262        logLine.put("message", generateLogMessage(event));
263        logLine.put("meta", generateLogMeta(event));
264        logLine.put("runtime", generateLogRuntime(event));
265        logLine.put("args", event.getArgumentArray());
266        if (event.getThrowableProxy() != null) {
267            logLine.put("throwable", generateLogThrowable(event.getThrowableProxy()));
268        }
269
270        return logLine;
271    }
272
273    protected String generateLogMessage(ILoggingEvent event) {
274        return this.encoder != null ? new String(this.encoder.encode(event)) : event.getFormattedMessage();
275    }
276
277    protected Map<String, Object> generateLogMeta(ILoggingEvent event) {
278        Map<String, Object> logMeta = new HashMap<>();
279        logMeta.put("logger", event.getLoggerName());
280
281        if (!mdcFields.isEmpty() && !event.getMDCPropertyMap().isEmpty()) {
282            for (Entry<String, String> entry : event.getMDCPropertyMap().entrySet()) {
283                if (mdcFields.contains(entry.getKey())) {
284                    String type = mdcTypes.get(mdcFields.indexOf(entry.getKey()));
285                    logMeta.put(entry.getKey(), getMetaValue(type, entry.getValue()));
286                }
287            }
288        }
289
290        return logMeta;
291    }
292
293    protected Map<String, Object> generateLogRuntime(ILoggingEvent event) {
294        Map<String, Object> logRuntime = new HashMap<>();
295        logRuntime.put("thread", event.getThreadName());
296
297        if (event.hasCallerData()) {
298            StackTraceElement[] callerData = event.getCallerData();
299
300            if (callerData.length > 0) {
301                StackTraceElement callerContext = callerData[0];
302
303                logRuntime.put("class", callerContext.getClassName());
304                logRuntime.put("method", callerContext.getMethodName());
305                logRuntime.put("file", callerContext.getFileName());
306                logRuntime.put("line", callerContext.getLineNumber());
307            }
308        }
309
310        return logRuntime;
311    }
312
313    protected Map<String, Object> generateLogThrowable(IThrowableProxy throwable) {
314        Map<String, Object> logThrowable = new HashMap<>();
315        logThrowable.put("message", throwable.getMessage());
316        logThrowable.put("class", throwable.getClassName());
317        logThrowable.put("stackTrace", throwable.getStackTraceElementProxyArray());
318        if (throwable.getCause() != null) {
319            logThrowable.put("cause", generateLogThrowable(throwable.getCause()));
320        }
321
322        return logThrowable;
323    }
324
325    protected Object getMetaValue(String type, String value) {
326        try {
327            switch (type) {
328                case "int":
329                    return Integer.valueOf(value);
330                case "long":
331                    return Long.valueOf(value);
332                case "boolean":
333                    return Boolean.valueOf(value);
334            }
335        } catch (NumberFormatException e) {
336            logger.error("Error getting meta value - {}", e.getMessage(), e);
337        }
338
339        return value;
340    }
341
342    public class LogfireSender implements Runnable {
343        @Override
344        public void run() {
345            try {
346                flush();
347            } catch (Exception e) {
348                logger.error("Error trying to flush : {}", e.getMessage(), e);
349                if (isFlushing.get()) {
350                    isFlushing.set(false);
351                }
352            }
353        }
354    }
355
356    /**
357     * Sets the application name for Logfire indexation.
358     *
359     * @param appName
360     *            application name
361     */
362    public void setAppName(String appName) {
363        this.appName = appName;
364    }
365
366    /**
367     * Sets the Logfire ingest API url.
368     *
369     * @param ingestUrl
370     *            Logfire ingest url
371     */
372    public void setIngestUrl(String ingestUrl) {
373        this.ingestUrl = ingestUrl;
374    }
375
376    /**
377     * Sets your Logfire source token.
378     *
379     * @param sourceToken
380     *            your Logfire source token
381     */
382    public void setSourceToken(String sourceToken) {
383        this.sourceToken = sourceToken;
384    }
385
386    /**
387     * Deprecated! Kept for backward compatibility.
388     * Sets your Logfire source token if unset.
389     *
390     * @param ingestKey
391     *            your Logfire source token
392     */
393    public void setIngestKey(String ingestKey) {
394        if (this.sourceToken == null) {
395            return;
396        }
397        this.sourceToken = ingestKey;
398    }
399
400    public void setUserAgent(String userAgent) {
401        this.userAgent = userAgent;
402    }
403
404    /**
405     * Sets the MDC fields that will be sent as metadata, separated by a comma.
406     *
407     * @param mdcFields
408     *            MDC fields to include in structured logs
409     */
410    public void setMdcFields(String mdcFields) {
411        this.mdcFields = Arrays.asList(mdcFields.split(","));
412    }
413
414    /**
415     * Sets the MDC fields types that will be sent as metadata, in the same order as <i>mdcFields</i> are set
416     * up, separated by a comma. Possible values are <i>string</i>, <i>boolean</i>, <i>int</i> and <i>long</i>.
417     *
418     * @param mdcTypes
419     *            MDC fields types
420     */
421    public void setMdcTypes(String mdcTypes) {
422        this.mdcTypes = Arrays.asList(mdcTypes.split(","));
423    }
424
425    /**
426     * Sets the maximum number of messages in the queue. Messages over the limit will be dropped.
427     *
428     * @param maxQueueSize
429     *            max size of the message queue
430     */
431    public void setMaxQueueSize(int maxQueueSize) {
432        this.maxQueueSize = maxQueueSize;
433    }
434
435    /**
436     * Sets the batch size for the number of messages to be sent via the API
437     *
438     * @param batchSize
439     *            size of the message batch
440     */
441    public void setBatchSize(int batchSize) {
442        this.batchSize = batchSize;
443    }
444
445    /**
446     * Get the batch size for the number of messages to be sent via the API
447     */
448    public int getBatchSize() {
449        return batchSize;
450    }
451
452    /**
453     * Sets the maximum wait time for a batch to be sent via the API, in milliseconds.
454     *
455     * @param batchInterval
456     *            maximum wait time for message batch [ms]
457     */
458    public void setBatchInterval(int batchInterval) {
459        scheduledFuture.cancel(false);
460        scheduledFuture = scheduledExecutorService.scheduleWithFixedDelay(new LogfireSender(), batchInterval, batchInterval, TimeUnit.MILLISECONDS);
461
462        this.batchInterval = batchInterval;
463    }
464
465    /**
466     * Sets the connection timeout of the underlying HTTP client, in milliseconds.
467     *
468     * @param connectTimeout
469     *            client connection timeout [ms]
470     */
471    public void setConnectTimeout(int connectTimeout) {
472        this.connectTimeout = connectTimeout;
473    }
474
475    /**
476     * Sets the read timeout of the underlying HTTP client, in milliseconds.
477     *
478     * @param readTimeout
479     *            client read timeout
480     */
481    public void setReadTimeout(int readTimeout) {
482        this.readTimeout = readTimeout;
483    }
484
485    /**
486     * Sets the maximum number of retries for sending logs to Logfire. After that, current batch of logs will be dropped.
487     *
488     * @param maxRetries
489     *            max number of retries for sending logs
490     */
491    public void setMaxRetries(int maxRetries) {
492        this.maxRetries = maxRetries;
493    }
494
495    /**
496     * Sets the number of milliseconds to sleep before retrying to send logs to Logfire.
497     *
498     * @param retrySleepMilliseconds
499     *            number of milliseconds to sleep before retry
500     */
501    public void setRetrySleepMilliseconds(int retrySleepMilliseconds) {
502        this.retrySleepMilliseconds = retrySleepMilliseconds;
503    }
504
505    /**
506     * Registers a dynamically loaded Module object to ObjectMapper used for serialization of logged data.
507     *
508     * @param className
509     *            fully qualified class name of the module, eg. "com.fasterxml.jackson.datatype.jsr310.JavaTimeModule"
510     */
511    public void setObjectMapperModule(String className) {
512        try {
513            Module module = (Module) Class.forName(className).newInstance();
514            dataMapper.registerModule(module);
515            logger.info("Module '{}' successfully registered in ObjectMapper.", className);
516        } catch (ClassNotFoundException|InstantiationException|IllegalAccessException e) {
517            logger.error("Module '{}' couldn't be registered in ObjectMapper : ", className, e);
518        }
519    }
520
521    public void setEncoder(PatternLayoutEncoder encoder) {
522        this.encoder = encoder;
523    }
524
525    public boolean isDisabled() {
526        return this.disabled;
527    }
528
529    @Override
530    public void stop() {
531        scheduledExecutorService.shutdown();
532        mustReflush = true;
533        flush();
534        super.stop();
535    }
536}