001package com.logfire.logback; 002 003import ch.qos.logback.classic.encoder.PatternLayoutEncoder; 004import ch.qos.logback.classic.spi.ILoggingEvent; 005import ch.qos.logback.classic.spi.IThrowableProxy; 006import ch.qos.logback.core.UnsynchronizedAppenderBase; 007import com.fasterxml.jackson.annotation.JsonInclude; 008import com.fasterxml.jackson.core.JsonProcessingException; 009import com.fasterxml.jackson.databind.Module; 010import com.fasterxml.jackson.databind.ObjectMapper; 011import com.fasterxml.jackson.databind.PropertyNamingStrategies; 012import org.slf4j.Logger; 013import org.slf4j.LoggerFactory; 014 015import java.io.IOException; 016import java.io.OutputStream; 017import java.net.HttpURLConnection; 018import java.net.URL; 019import java.nio.charset.StandardCharsets; 020import java.util.*; 021import java.util.Map.Entry; 022import java.util.concurrent.atomic.AtomicBoolean; 023import java.util.concurrent.Executors; 024import java.util.concurrent.ScheduledExecutorService; 025import java.util.concurrent.ScheduledFuture; 026import java.util.concurrent.ThreadFactory; 027import java.util.concurrent.TimeUnit; 028import java.util.stream.Collectors; 029 030public class LogfireAppender extends UnsynchronizedAppenderBase<ILoggingEvent> { 031 032 // Customizable variables 033 protected String appName; 034 protected String ingestUrl = "https://in.logfire.ai"; 035 036 protected String sourceToken; 037 protected String userAgent = "Logfire Logback Appender"; 038 039 protected List<String> mdcFields = new ArrayList<>(); 040 protected List<String> mdcTypes = new ArrayList<>(); 041 042 protected int maxQueueSize = 100000; 043 protected int batchSize = 1000; 044 protected int batchInterval = 3000; 045 protected int connectTimeout = 5000; 046 protected int readTimeout = 10000; 047 protected int maxRetries = 5; 048 protected int retrySleepMilliseconds = 300; 049 050 protected PatternLayoutEncoder encoder; 051 052 // Non-customizable variables 053 protected Vector<ILoggingEvent> batch = new Vector<>(); 054 protected AtomicBoolean isFlushing = new AtomicBoolean(false); 055 protected boolean mustReflush = false; 056 protected boolean warnAboutMaxQueueSize = true; 057 058 // Utils 059 protected ScheduledExecutorService scheduledExecutorService; 060 protected ScheduledFuture<?> scheduledFuture; 061 protected ObjectMapper dataMapper; 062 protected Logger logger; 063 protected int retrySize = 0; 064 protected int retries = 0; 065 protected boolean disabled = false; 066 067 protected ThreadFactory threadFactory = r -> { 068 Thread thread = Executors.defaultThreadFactory().newThread(r); 069 thread.setName("logfire-appender"); 070 thread.setDaemon(true); 071 return thread; 072 }; 073 074 public LogfireAppender() { 075 logger = LoggerFactory.getLogger(LogfireAppender.class); 076 077 dataMapper = new ObjectMapper() 078 .setSerializationInclusion(JsonInclude.Include.NON_NULL) 079 .setPropertyNamingStrategy(PropertyNamingStrategies.UPPER_CAMEL_CASE); 080 081 scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(threadFactory); 082 scheduledFuture = scheduledExecutorService.scheduleWithFixedDelay(new LogfireSender(), batchInterval, batchInterval, TimeUnit.MILLISECONDS); 083 } 084 085 @Override 086 protected void append(ILoggingEvent event) { 087 if (disabled) 088 return; 089 090 if (event.getLoggerName().equals(LogfireAppender.class.getName())) 091 return; 092 093 if (this.ingestUrl.isEmpty() || this.sourceToken == null || this.sourceToken.isEmpty()) { 094 // Prevent potential deadlock, when a blocking logger is configured - avoid using logger directly in append 095 startThread("logfire-warning-logger", () -> { 096 logger.warn("Missing Source token for Logfire - disabling LogfireAppender. Find out how to fix this at: https://logfire.ai/docs/logs/java "); 097 }); 098 this.disabled = true; 099 return; 100 } 101 102 if (batch.size() < maxQueueSize) { 103 batch.add(event); 104 } 105 106 if (warnAboutMaxQueueSize && batch.size() == maxQueueSize) { 107 this.warnAboutMaxQueueSize = false; 108 // Prevent potential deadlock, when a blocking logger is configured - avoid using logger directly in append 109 startThread("logfire-error-logger", () -> { 110 logger.error("Maximum number of messages in queue reached ({}). New messages will be dropped.", maxQueueSize); 111 }); 112 } 113 114 if (batch.size() >= batchSize) { 115 if (isFlushing.get()) 116 return; 117 118 startThread("logfire-appender-flush", new LogfireSender()); 119 } 120 } 121 122 protected void startThread(String threadName, Runnable runnable) { 123 Thread thread = Executors.defaultThreadFactory().newThread(runnable); 124 thread.setName(threadName); 125 thread.start(); 126 } 127 128 protected void flush() { 129 if (batch.isEmpty()) 130 return; 131 132 // Guaranteed to not be running concurrently 133 if (isFlushing.getAndSet(true)) 134 return; 135 136 mustReflush = false; 137 138 int flushedSize = batch.size(); 139 if (flushedSize > batchSize) { 140 flushedSize = batchSize; 141 mustReflush = true; 142 } 143 if (retries > 0 && flushedSize > retrySize) { 144 flushedSize = retrySize; 145 mustReflush = true; 146 } 147 148 if (!flushLogs(flushedSize)) { 149 mustReflush = true; 150 } 151 152 isFlushing.set(false); 153 154 if (mustReflush || batch.size() >= batchSize) { 155 flush(); 156 } 157 } 158 159 protected boolean flushLogs(int flushedSize) { 160 retrySize = flushedSize; 161 162 try { 163 if (retries > maxRetries) { 164 batch.subList(0, flushedSize).clear(); 165 logger.error("Dropped batch of {} logs.", flushedSize); 166 warnAboutMaxQueueSize = true; 167 retries = 0; 168 169 return true; 170 } 171 172 if (retries > 0) { 173 logger.info("Retrying to send {} logs to Logfire ({} / {})", flushedSize, retries, maxRetries); 174 try { 175 TimeUnit.MILLISECONDS.sleep(retrySleepMilliseconds); 176 } catch (InterruptedException e) { 177 // Continue 178 } 179 } 180 181 LogfireResponse response = callHttpURLConnection(flushedSize); 182 183 if (response.getStatus() >= 300 || response.getStatus() < 200) { 184 logger.error("Error calling Logfire : {} ({})", response.getError(), response.getStatus()); 185 retries++; 186 187 return false; 188 } 189 190 batch.subList(0, flushedSize).clear(); 191 warnAboutMaxQueueSize = true; 192 retries = 0; 193 194 return true; 195 196 } catch (ConcurrentModificationException e) { 197 logger.error("Error clearing {} logs from batch, will retry immediately.", flushedSize, e); 198 retries = maxRetries; // No point in retrying to send the data 199 200 } catch (JsonProcessingException e) { 201 logger.error("Error processing JSON data : {}", e.getMessage(), e); 202 retries = maxRetries; // No point in retrying when batch cannot be processed into JSON 203 204 } catch (Exception e) { 205 logger.error("Error trying to call Logfire : {}", e.getMessage(), e); 206 } 207 208 retries++; 209 210 return false; 211 } 212 213 protected LogfireResponse callHttpURLConnection(int flushedSize) throws IOException { 214 HttpURLConnection connection = getHttpURLConnection(); 215 216 try { 217 connection.connect(); 218 } catch (Exception e) { 219 logger.error("Error trying to call Logfire : {}", e.getMessage(), e); 220 } 221 222 try (OutputStream os = connection.getOutputStream()) { 223 byte[] input = batchToJson(flushedSize).getBytes(StandardCharsets.UTF_8); 224 os.write(input, 0, input.length); 225 os.flush(); 226 } 227 228 connection.disconnect(); 229 230 return new LogfireResponse(connection.getResponseMessage(), connection.getResponseCode()); 231 } 232 233 protected HttpURLConnection getHttpURLConnection() throws IOException { 234 HttpURLConnection httpURLConnection = (HttpURLConnection) new URL(this.ingestUrl).openConnection(); 235 httpURLConnection.setDoOutput(true); 236 httpURLConnection.setDoInput(true); 237 httpURLConnection.setRequestProperty("User-Agent", this.userAgent); 238 httpURLConnection.setRequestProperty("Accept", "application/json"); 239 httpURLConnection.setRequestProperty("Content-Type", "application/json"); 240 httpURLConnection.setRequestProperty("Charset", "UTF-8"); 241 httpURLConnection.setRequestProperty("Authorization", String.format("Bearer %s", this.sourceToken)); 242 httpURLConnection.setRequestMethod("POST"); 243 httpURLConnection.setConnectTimeout(this.connectTimeout); 244 httpURLConnection.setReadTimeout(this.readTimeout); 245 return httpURLConnection; 246 } 247 248 protected String batchToJson(int flushedSize) throws JsonProcessingException { 249 return this.dataMapper.writeValueAsString( 250 new ArrayList<>(batch.subList(0, flushedSize)) 251 .stream() 252 .map(this::buildPostData) 253 .collect(Collectors.toList()) 254 ); 255 } 256 257 protected Map<String, Object> buildPostData(ILoggingEvent event) { 258 Map<String, Object> logLine = new HashMap<>(); 259 logLine.put("dt", Long.toString(event.getTimeStamp())); 260 logLine.put("level", event.getLevel().toString()); 261 logLine.put("app", this.appName); 262 logLine.put("message", generateLogMessage(event)); 263 logLine.put("meta", generateLogMeta(event)); 264 logLine.put("runtime", generateLogRuntime(event)); 265 logLine.put("args", event.getArgumentArray()); 266 if (event.getThrowableProxy() != null) { 267 logLine.put("throwable", generateLogThrowable(event.getThrowableProxy())); 268 } 269 270 return logLine; 271 } 272 273 protected String generateLogMessage(ILoggingEvent event) { 274 return this.encoder != null ? new String(this.encoder.encode(event)) : event.getFormattedMessage(); 275 } 276 277 protected Map<String, Object> generateLogMeta(ILoggingEvent event) { 278 Map<String, Object> logMeta = new HashMap<>(); 279 logMeta.put("logger", event.getLoggerName()); 280 281 if (!mdcFields.isEmpty() && !event.getMDCPropertyMap().isEmpty()) { 282 for (Entry<String, String> entry : event.getMDCPropertyMap().entrySet()) { 283 if (mdcFields.contains(entry.getKey())) { 284 String type = mdcTypes.get(mdcFields.indexOf(entry.getKey())); 285 logMeta.put(entry.getKey(), getMetaValue(type, entry.getValue())); 286 } 287 } 288 } 289 290 return logMeta; 291 } 292 293 protected Map<String, Object> generateLogRuntime(ILoggingEvent event) { 294 Map<String, Object> logRuntime = new HashMap<>(); 295 logRuntime.put("thread", event.getThreadName()); 296 297 if (event.hasCallerData()) { 298 StackTraceElement[] callerData = event.getCallerData(); 299 300 if (callerData.length > 0) { 301 StackTraceElement callerContext = callerData[0]; 302 303 logRuntime.put("class", callerContext.getClassName()); 304 logRuntime.put("method", callerContext.getMethodName()); 305 logRuntime.put("file", callerContext.getFileName()); 306 logRuntime.put("line", callerContext.getLineNumber()); 307 } 308 } 309 310 return logRuntime; 311 } 312 313 protected Map<String, Object> generateLogThrowable(IThrowableProxy throwable) { 314 Map<String, Object> logThrowable = new HashMap<>(); 315 logThrowable.put("message", throwable.getMessage()); 316 logThrowable.put("class", throwable.getClassName()); 317 logThrowable.put("stackTrace", throwable.getStackTraceElementProxyArray()); 318 if (throwable.getCause() != null) { 319 logThrowable.put("cause", generateLogThrowable(throwable.getCause())); 320 } 321 322 return logThrowable; 323 } 324 325 protected Object getMetaValue(String type, String value) { 326 try { 327 switch (type) { 328 case "int": 329 return Integer.valueOf(value); 330 case "long": 331 return Long.valueOf(value); 332 case "boolean": 333 return Boolean.valueOf(value); 334 } 335 } catch (NumberFormatException e) { 336 logger.error("Error getting meta value - {}", e.getMessage(), e); 337 } 338 339 return value; 340 } 341 342 public class LogfireSender implements Runnable { 343 @Override 344 public void run() { 345 try { 346 flush(); 347 } catch (Exception e) { 348 logger.error("Error trying to flush : {}", e.getMessage(), e); 349 if (isFlushing.get()) { 350 isFlushing.set(false); 351 } 352 } 353 } 354 } 355 356 /** 357 * Sets the application name for Logfire indexation. 358 * 359 * @param appName 360 * application name 361 */ 362 public void setAppName(String appName) { 363 this.appName = appName; 364 } 365 366 /** 367 * Sets the Logfire ingest API url. 368 * 369 * @param ingestUrl 370 * Logfire ingest url 371 */ 372 public void setIngestUrl(String ingestUrl) { 373 this.ingestUrl = ingestUrl; 374 } 375 376 /** 377 * Sets your Logfire source token. 378 * 379 * @param sourceToken 380 * your Logfire source token 381 */ 382 public void setSourceToken(String sourceToken) { 383 this.sourceToken = sourceToken; 384 } 385 386 /** 387 * Deprecated! Kept for backward compatibility. 388 * Sets your Logfire source token if unset. 389 * 390 * @param ingestKey 391 * your Logfire source token 392 */ 393 public void setIngestKey(String ingestKey) { 394 if (this.sourceToken == null) { 395 return; 396 } 397 this.sourceToken = ingestKey; 398 } 399 400 public void setUserAgent(String userAgent) { 401 this.userAgent = userAgent; 402 } 403 404 /** 405 * Sets the MDC fields that will be sent as metadata, separated by a comma. 406 * 407 * @param mdcFields 408 * MDC fields to include in structured logs 409 */ 410 public void setMdcFields(String mdcFields) { 411 this.mdcFields = Arrays.asList(mdcFields.split(",")); 412 } 413 414 /** 415 * Sets the MDC fields types that will be sent as metadata, in the same order as <i>mdcFields</i> are set 416 * up, separated by a comma. Possible values are <i>string</i>, <i>boolean</i>, <i>int</i> and <i>long</i>. 417 * 418 * @param mdcTypes 419 * MDC fields types 420 */ 421 public void setMdcTypes(String mdcTypes) { 422 this.mdcTypes = Arrays.asList(mdcTypes.split(",")); 423 } 424 425 /** 426 * Sets the maximum number of messages in the queue. Messages over the limit will be dropped. 427 * 428 * @param maxQueueSize 429 * max size of the message queue 430 */ 431 public void setMaxQueueSize(int maxQueueSize) { 432 this.maxQueueSize = maxQueueSize; 433 } 434 435 /** 436 * Sets the batch size for the number of messages to be sent via the API 437 * 438 * @param batchSize 439 * size of the message batch 440 */ 441 public void setBatchSize(int batchSize) { 442 this.batchSize = batchSize; 443 } 444 445 /** 446 * Get the batch size for the number of messages to be sent via the API 447 */ 448 public int getBatchSize() { 449 return batchSize; 450 } 451 452 /** 453 * Sets the maximum wait time for a batch to be sent via the API, in milliseconds. 454 * 455 * @param batchInterval 456 * maximum wait time for message batch [ms] 457 */ 458 public void setBatchInterval(int batchInterval) { 459 scheduledFuture.cancel(false); 460 scheduledFuture = scheduledExecutorService.scheduleWithFixedDelay(new LogfireSender(), batchInterval, batchInterval, TimeUnit.MILLISECONDS); 461 462 this.batchInterval = batchInterval; 463 } 464 465 /** 466 * Sets the connection timeout of the underlying HTTP client, in milliseconds. 467 * 468 * @param connectTimeout 469 * client connection timeout [ms] 470 */ 471 public void setConnectTimeout(int connectTimeout) { 472 this.connectTimeout = connectTimeout; 473 } 474 475 /** 476 * Sets the read timeout of the underlying HTTP client, in milliseconds. 477 * 478 * @param readTimeout 479 * client read timeout 480 */ 481 public void setReadTimeout(int readTimeout) { 482 this.readTimeout = readTimeout; 483 } 484 485 /** 486 * Sets the maximum number of retries for sending logs to Logfire. After that, current batch of logs will be dropped. 487 * 488 * @param maxRetries 489 * max number of retries for sending logs 490 */ 491 public void setMaxRetries(int maxRetries) { 492 this.maxRetries = maxRetries; 493 } 494 495 /** 496 * Sets the number of milliseconds to sleep before retrying to send logs to Logfire. 497 * 498 * @param retrySleepMilliseconds 499 * number of milliseconds to sleep before retry 500 */ 501 public void setRetrySleepMilliseconds(int retrySleepMilliseconds) { 502 this.retrySleepMilliseconds = retrySleepMilliseconds; 503 } 504 505 /** 506 * Registers a dynamically loaded Module object to ObjectMapper used for serialization of logged data. 507 * 508 * @param className 509 * fully qualified class name of the module, eg. "com.fasterxml.jackson.datatype.jsr310.JavaTimeModule" 510 */ 511 public void setObjectMapperModule(String className) { 512 try { 513 Module module = (Module) Class.forName(className).newInstance(); 514 dataMapper.registerModule(module); 515 logger.info("Module '{}' successfully registered in ObjectMapper.", className); 516 } catch (ClassNotFoundException|InstantiationException|IllegalAccessException e) { 517 logger.error("Module '{}' couldn't be registered in ObjectMapper : ", className, e); 518 } 519 } 520 521 public void setEncoder(PatternLayoutEncoder encoder) { 522 this.encoder = encoder; 523 } 524 525 public boolean isDisabled() { 526 return this.disabled; 527 } 528 529 @Override 530 public void stop() { 531 scheduledExecutorService.shutdown(); 532 mustReflush = true; 533 flush(); 534 super.stop(); 535 } 536}