feat(core): [Scope-Based Deduplication 3] Add scope-based deduplication option

CHANGELOG.md

@@ -4,6 +4,9 @@
 
 ### Features
 
+- Added experimental scope-based duplicate event detection for reusable exception instances ([#6431](https://github.com/getsentry/sentry-java/pull/6431))
+  - Enable it with `enable-deduplication=false` and `enable-scope-based-deduplication=true`, or the equivalent `SentryOptions` setters.
+  - Deduplication state is isolated per isolation scope, so request integrations that fork scopes no longer suppress later events when one exception instance is reused.
 - Added `Environment` to the `Scope`, allowing you to overwrite the `Environment` set on the options ([#5772](https://github.com/getsentry/sentry-java/pull/5772))
   - The scope `Environment` is also applied to the trace envelope header (DSC), logs, metrics and profiles, so all telemetry of a trace reports the same environment
 

sentry-android-core/src/main/java/io/sentry/android/core/ManifestMetadataReader.java

@@ -141,6 +141,9 @@ final class ManifestMetadataReader {
 
   static final String ENABLE_SENTRY = "io.sentry.enabled";
 
+  static final String ENABLE_SCOPE_BASED_DEDUPLICATION =
+      "io.sentry.enable-scope-based-deduplication";
+
   static final String SEND_MODULES = "io.sentry.send-modules";
 
   static final String ENABLE_PERFORMANCE_V2 = "io.sentry.performance-v2.enable";
@@ -357,6 +360,12 @@ static void applyMetadata(
 
         options.setEnabled(enabled);
         options.setDsn(dsn);
+        options.setEnableScopeBasedDeduplication(
+            readBool(
+                metadata,
+                logger,
+                ENABLE_SCOPE_BASED_DEDUPLICATION,
+                options.isEnableScopeBasedDeduplication()));
 
         options.setEnableNdk(readBool(metadata, logger, NDK_ENABLE, options.isEnableNdk()));
 

sentry-android-core/src/test/java/io/sentry/android/core/ManifestMetadataReaderTest.kt

@@ -84,6 +84,40 @@ class ManifestMetadataReaderTest {
     verifyNoInteractions(context)
   }
 
+  @Test
+  fun `applyMetadata enables scope-based deduplication from manifest`() {
+    val context =
+      fixture.getContext(bundleOf(ManifestMetadataReader.ENABLE_SCOPE_BASED_DEDUPLICATION to true))
+
+    ManifestMetadataReader.applyMetadata(context, fixture.options, fixture.buildInfoProvider)
+
+    assertThat(fixture.options.isEnableScopeBasedDeduplication).isTrue()
+  }
+
+  @Test
+  fun `applyMetadata disables scope-based deduplication from manifest`() {
+    fixture.options.isEnableScopeBasedDeduplication = true
+    val context =
+      fixture.getContext(bundleOf(ManifestMetadataReader.ENABLE_SCOPE_BASED_DEDUPLICATION to false))
+
+    ManifestMetadataReader.applyMetadata(context, fixture.options, fixture.buildInfoProvider)
+
+    assertThat(fixture.options.isEnableScopeBasedDeduplication).isFalse()
+  }
+
+  @Test
+  fun `applyMetadata preserves scope-based deduplication when manifest value is absent`() {
+    fixture.options.isEnableScopeBasedDeduplication = true
+
+    ManifestMetadataReader.applyMetadata(
+      fixture.getContext(),
+      fixture.options,
+      fixture.buildInfoProvider,
+    )
+
+    assertThat(fixture.options.isEnableScopeBasedDeduplication).isTrue()
+  }
+
   @Test
   fun `isAutoInit reads build-time metadata without querying context`() {
     val context = mock<Context>()

sentry-android-core/src/test/java/io/sentry/android/core/SentryAndroidTest.kt

@@ -213,6 +213,30 @@ class SentryAndroidTest {
     assertTrue(Sentry.isEnabled())
   }
 
+  @Test
+  fun `manifest scope-based deduplication is applied before processor installation`() {
+    ContextUtils.resetInstance()
+    val metadata =
+      Bundle().apply {
+        putString(ManifestMetadataReader.DSN, "https://key@sentry.io/123")
+        putBoolean(ManifestMetadataReader.ENABLE_SCOPE_BASED_DEDUPLICATION, true)
+      }
+
+    fixture.initSut(ContextUtilsTestHelper.mockMetaData(metaData = metadata)) {
+      it.isEnableDeduplication = false
+    }
+
+    val scopes = Sentry.getCurrentScopes()
+    assertFalse(scopes.options.isEnableDeduplication)
+    assertTrue(scopes.options.isEnableScopeBasedDeduplication)
+    assertEquals(
+      1,
+      scopes.isolationScope.eventProcessors.count {
+        it.javaClass.simpleName == "ScopeBasedDuplicateEventDetectionEventProcessor"
+      },
+    )
+  }
+
   @Test
   fun `init won't throw exception`() {
     val logger = mock<ILogger>()

sentry-jul/src/test/kotlin/io/sentry/jul/SentryHandlerTest.kt

@@ -28,6 +28,7 @@ import kotlin.test.assertNull
 import kotlin.test.assertTrue
 import org.mockito.kotlin.anyOrNull
 import org.mockito.kotlin.mock
+import org.mockito.kotlin.times
 import org.mockito.kotlin.verify
 import org.slf4j.MDC
 
@@ -40,6 +41,8 @@ class SentryHandlerTest {
     val transport: ITransport = mock(),
     contextTags: List<String>? = null,
     printfStyle: Boolean? = null,
+    enableDeduplication: Boolean = true,
+    enableScopeBasedDeduplication: Boolean = false,
   ) {
     var logger: Logger
     var handler: SentryHandler
@@ -48,6 +51,8 @@ class SentryHandlerTest {
       val options = SentryOptions()
       options.dsn = "http://key@localhost/proj"
       options.setTransportFactory { _, _ -> transport }
+      options.isEnableDeduplication = enableDeduplication
+      options.isEnableScopeBasedDeduplication = enableScopeBasedDeduplication
       options.logs.loggerBatchProcessorFactory = ILoggerBatchProcessorFactory { options, client ->
         LoggerBatchProcessor(options, client, ImmediateExecutorService())
       }
@@ -478,6 +483,28 @@ class SentryHandlerTest {
       .send(checkLogs { event -> assertEquals(SentryLogLevel.WARN, event.items.first().level) })
   }
 
+  @Test
+  fun `scope-based deduplication sends repeated reusable exceptions from JUL`() {
+    fixture =
+      Fixture(
+        minimumEventLevel = Level.SEVERE,
+        enableDeduplication = false,
+        enableScopeBasedDeduplication = true,
+      )
+    val throwable = RuntimeException("reused")
+
+    fixture.logger.log(Level.SEVERE, "first", throwable)
+    fixture.logger.log(Level.SEVERE, "second", throwable)
+
+    verify(fixture.transport, times(2))
+      .send(
+        checkEvent { event ->
+          assertEquals(SentryHandler.MECHANISM_TYPE, event.exceptions!!.first().mechanism!!.type)
+        },
+        anyOrNull(),
+      )
+  }
+
   @Test
   fun `converts severe log level to Sentry log level`() {
     fixture = Fixture(minimumLevel = Level.SEVERE)

sentry-log4j2/src/test/kotlin/io/sentry/log4j2/SentryAppenderTest.kt

@@ -36,6 +36,7 @@ import org.apache.logging.log4j.spi.ExtendedLogger
 import org.mockito.kotlin.any
 import org.mockito.kotlin.anyOrNull
 import org.mockito.kotlin.mock
+import org.mockito.kotlin.times
 import org.mockito.kotlin.verify
 import org.mockito.kotlin.whenever
 
@@ -57,6 +58,8 @@ class SentryAppenderTest {
       minimumLevel: Level? = null,
       debug: Boolean? = null,
       contextTags: List<String>? = null,
+      enableDeduplication: Boolean = true,
+      enableScopeBasedDeduplication: Boolean = false,
     ): ExtendedLogger {
       if (transportFactory != null) {
         this.transportFactory = transportFactory
@@ -98,6 +101,8 @@ class SentryAppenderTest {
 
       appender.start(
         appender.getOptionsConfiguration { options ->
+          options.isEnableDeduplication = enableDeduplication
+          options.isEnableScopeBasedDeduplication = enableScopeBasedDeduplication
           options.logs.loggerBatchProcessorFactory =
             ILoggerBatchProcessorFactory { options, client ->
               LoggerBatchProcessor(options, client, ImmediateExecutorService())
@@ -162,6 +167,28 @@ class SentryAppenderTest {
       )
   }
 
+  @Test
+  fun `scope-based deduplication sends repeated reusable exceptions from Log4j2`() {
+    val logger =
+      fixture.getSut(
+        minimumEventLevel = Level.ERROR,
+        enableDeduplication = false,
+        enableScopeBasedDeduplication = true,
+      )
+    val throwable = RuntimeException("reused")
+
+    logger.error("first", throwable)
+    logger.error("second", throwable)
+
+    verify(fixture.transport, times(2))
+      .send(
+        checkEvent { event ->
+          assertEquals(SentryAppender.MECHANISM_TYPE, event.exceptions!!.first().mechanism!!.type)
+        },
+        anyOrNull(),
+      )
+  }
+
   @Test
   fun `event date is in UTC`() {
     val logger = fixture.getSut(minimumEventLevel = Level.DEBUG)

sentry-logback/src/test/kotlin/io/sentry/logback/SentryAppenderTest.kt

@@ -39,6 +39,7 @@ import kotlin.test.assertTrue
 import org.mockito.kotlin.any
 import org.mockito.kotlin.anyOrNull
 import org.mockito.kotlin.mock
+import org.mockito.kotlin.times
 import org.mockito.kotlin.verify
 import org.mockito.kotlin.whenever
 import org.slf4j.Logger
@@ -410,6 +411,31 @@ class SentryAppenderTest {
       )
   }
 
+  @Test
+  fun `scope-based deduplication sends repeated reusable exceptions from Logback`() {
+    fixture =
+      Fixture(
+        minimumEventLevel = Level.ERROR,
+        options =
+          SentryOptions().apply {
+            isEnableDeduplication = false
+            isEnableScopeBasedDeduplication = true
+          },
+      )
+    val throwable = RuntimeException("reused")
+
+    fixture.logger.error("first", throwable)
+    fixture.logger.error("second", throwable)
+
+    verify(fixture.transport, times(2))
+      .send(
+        checkEvent { event ->
+          assertEquals(SentryAppender.MECHANISM_TYPE, event.exceptions!!.first().mechanism!!.type)
+        },
+        anyOrNull(),
+      )
+  }
+
   @Test
   fun `converts trace log level to Sentry log level`() {
     fixture = Fixture(minimumLevel = Level.TRACE, enableLogs = true)

sentry-spring-boot-4/src/test/kotlin/io/sentry/spring/boot4/SentryAutoConfigurationTest.kt

@@ -148,6 +148,25 @@ class SentryAutoConfigurationTest {
     dsnEnabledRunner.run { assertThat(it).hasSingleBean(IScopes::class.java) }
   }
 
+  @Test
+  fun `scope-based deduplication property is applied before processor installation`() {
+    dsnEnabledRunner
+      .withPropertyValues(
+        "sentry.enable-deduplication=false",
+        "sentry.enable-scope-based-deduplication=true",
+      )
+      .run {
+        val scopes = it.getBean(IScopes::class.java)
+        assertThat(scopes.options.isEnableScopeBasedDeduplication).isTrue()
+        assertThat(
+            scopes.isolationScope.eventProcessors.filter { processor ->
+              processor.javaClass.simpleName == "ScopeBasedDuplicateEventDetectionEventProcessor"
+            }
+          )
+          .hasSize(1)
+      }
+  }
+
   @Test
   fun `OptionsConfiguration is created if custom one with name sentryOptionsConfiguration is not provided`() {
     dsnEnabledRunner.run { assertThat(it).hasSingleBean(Sentry.OptionsConfiguration::class.java) }
@@ -204,6 +223,7 @@ class SentryAutoConfigurationTest {
         "sentry.shutdown-timeout-millis=20",
         "sentry.flush-timeout-millis=30",
         "sentry.debug=true",
+        "sentry.enable-scope-based-deduplication=true",
         "sentry.diagnostic-level=INFO",
         "sentry.sentry-client-name=my-client",
         "sentry.max-breadcrumbs=100",
@@ -254,6 +274,7 @@ class SentryAutoConfigurationTest {
         assertThat(options.shutdownTimeoutMillis).isEqualTo(20)
         assertThat(options.flushTimeoutMillis).isEqualTo(30)
         assertThat(options.isDebug).isTrue()
+        assertThat(options.isEnableScopeBasedDeduplication).isTrue()
         assertThat(options.diagnosticLevel).isEqualTo(SentryLevel.INFO)
         assertThat(options.maxBreadcrumbs).isEqualTo(100)
         assertThat(options.release).isEqualTo("1.0.3")

sentry-spring-boot-jakarta/src/test/kotlin/io/sentry/spring/boot/jakarta/SentryAutoConfigurationTest.kt

@@ -151,6 +151,25 @@ class SentryAutoConfigurationTest {
     dsnEnabledRunner.run { assertThat(it).hasSingleBean(IScopes::class.java) }
   }
 
+  @Test
+  fun `scope-based deduplication property is applied before processor installation`() {
+    dsnEnabledRunner
+      .withPropertyValues(
+        "sentry.enable-deduplication=false",
+        "sentry.enable-scope-based-deduplication=true",
+      )
+      .run {
+        val scopes = it.getBean(IScopes::class.java)
+        assertThat(scopes.options.isEnableScopeBasedDeduplication).isTrue()
+        assertThat(
+            scopes.isolationScope.eventProcessors.filter { processor ->
+              processor.javaClass.simpleName == "ScopeBasedDuplicateEventDetectionEventProcessor"
+            }
+          )
+          .hasSize(1)
+      }
+  }
+
   @Test
   fun `OptionsConfiguration is created if custom one with name sentryOptionsConfiguration is not provided`() {
     dsnEnabledRunner.run { assertThat(it).hasSingleBean(Sentry.OptionsConfiguration::class.java) }
@@ -207,6 +226,7 @@ class SentryAutoConfigurationTest {
         "sentry.shutdown-timeout-millis=20",
         "sentry.flush-timeout-millis=30",
         "sentry.debug=true",
+        "sentry.enable-scope-based-deduplication=true",
         "sentry.diagnostic-level=INFO",
         "sentry.sentry-client-name=my-client",
         "sentry.max-breadcrumbs=100",
@@ -259,6 +279,7 @@ class SentryAutoConfigurationTest {
         assertThat(options.shutdownTimeoutMillis).isEqualTo(20)
         assertThat(options.flushTimeoutMillis).isEqualTo(30)
         assertThat(options.isDebug).isTrue()
+        assertThat(options.isEnableScopeBasedDeduplication).isTrue()
         assertThat(options.diagnosticLevel).isEqualTo(SentryLevel.INFO)
         assertThat(options.maxBreadcrumbs).isEqualTo(100)
         assertThat(options.release).isEqualTo("1.0.3")

sentry-spring-boot/src/test/kotlin/io/sentry/spring/boot/SentryAutoConfigurationTest.kt

@@ -150,6 +150,25 @@ class SentryAutoConfigurationTest {
     dsnEnabledRunner.run { assertThat(it).hasSingleBean(IScopes::class.java) }
   }
 
+  @Test
+  fun `scope-based deduplication property is applied before processor installation`() {
+    dsnEnabledRunner
+      .withPropertyValues(
+        "sentry.enable-deduplication=false",
+        "sentry.enable-scope-based-deduplication=true",
+      )
+      .run {
+        val scopes = it.getBean(IScopes::class.java)
+        assertThat(scopes.options.isEnableScopeBasedDeduplication).isTrue()
+        assertThat(
+            scopes.isolationScope.eventProcessors.filter { processor ->
+              processor.javaClass.simpleName == "ScopeBasedDuplicateEventDetectionEventProcessor"
+            }
+          )
+          .hasSize(1)
+      }
+  }
+
   @Test
   fun `OptionsConfiguration is created if custom one with name sentryOptionsConfiguration is not provided`() {
     dsnEnabledRunner.run { assertThat(it).hasSingleBean(Sentry.OptionsConfiguration::class.java) }
@@ -205,6 +224,7 @@ class SentryAutoConfigurationTest {
         "sentry.shutdown-timeout-millis=20",
         "sentry.flush-timeout-millis=30",
         "sentry.debug=true",
+        "sentry.enable-scope-based-deduplication=true",
         "sentry.diagnostic-level=INFO",
         "sentry.sentry-client-name=my-client",
         "sentry.max-breadcrumbs=100",
@@ -257,6 +277,7 @@ class SentryAutoConfigurationTest {
         assertThat(options.shutdownTimeoutMillis).isEqualTo(20)
         assertThat(options.flushTimeoutMillis).isEqualTo(30)
         assertThat(options.isDebug).isTrue()
+        assertThat(options.isEnableScopeBasedDeduplication).isTrue()
         assertThat(options.diagnosticLevel).isEqualTo(SentryLevel.INFO)
         assertThat(options.maxBreadcrumbs).isEqualTo(100)
         assertThat(options.release).isEqualTo("1.0.3")