Add support for Spark 3.1 and 3.2

buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/BuildPlugin.groovy

@@ -314,11 +314,6 @@ class BuildPlugin implements Plugin<Project>  {
                     if (details.requested.name.contains("org.apache.logging.log4j") && details.requested.name.contains("log4j-")) {
                         details.useVersion project.ext.log4jVersion
                     }
-                    // Convert any references to the servlet-api into the jetty servlet artifact.
-                    if (details.requested.name.equals("servlet-api")) {
-                        details.useTarget group: "org.eclipse.jetty.orbit", name: "javax.servlet", version: "3.0.0.v201112011016"
-                    }
-
                 }
             }
         }

gradle.properties

@@ -30,7 +30,7 @@ spark13Version = 1.6.2
 spark20Version = 2.3.0
 spark22Version = 2.2.3
 spark24Version = 2.4.4
-spark30Version = 3.0.1
+spark30Version = 3.2.0
 
 # same as Spark's
 scala210Version = 2.10.7

licenses/spark-catalyst_2.12-3.2.0.jar.sha1

@@ -0,0 +1 @@
+6d142b9edf57e3a6dddd26df7e7ff2b3dc860802

licenses/spark-core_2.12-3.2.0.jar.sha1

@@ -0,0 +1 @@
+4202b5b58063aacb54fa37a876df18472a980267

licenses/spark-sql_2.12-3.2.0.jar.sha1

@@ -0,0 +1 @@
+0a366219179e0354379d8c965edabc9666b99e09

licenses/spark-streaming_2.12-3.2.0.jar.sha1

@@ -0,0 +1 @@
+deae68c8374bdf4d7da6943d3fd3a39c8fd04dcf

licenses/spark-yarn_2.12-3.2.0.jar.sha1

@@ -0,0 +1 @@
+34a823614c0d46d534c3995cb5c94348c1377c70

spark/sql-30/build.gradle

@@ -84,6 +84,7 @@ sparkVariants {
             add(variant.configuration('compileOnly'), "org.apache.spark:spark-tags_${variant.scalaMajorVersion}:$variant.sparkVersion")
 
             add(variant.configuration('test', 'implementation'), project(":test:shared"))
+            add(variant.configuration('test', 'implementation'), "jakarta.servlet:jakarta.servlet-api:4.0.3")
             add(variant.configuration('test', 'implementation'), "org.elasticsearch:securemock:1.2")
             add(variant.configuration('test', 'implementation'), "org.apache.spark:spark-core_${variant.scalaMajorVersion}:$variant.sparkVersion") {
                 exclude group: 'javax.servlet'
@@ -94,6 +95,7 @@ sparkVariants {
             }
 
             add(variant.configuration('itest', 'implementation'), project(":test:shared"))
+            add(variant.configuration('itest', 'implementation'), "jakarta.servlet:jakarta.servlet-api:4.0.3")
             add(variant.configuration('itest', 'implementation'), "org.apache.spark:spark-yarn_${variant.scalaMajorVersion}:$variant.sparkVersion") {
                 exclude group: 'org.apache.hadoop'
             }
@@ -159,16 +161,16 @@ sparkVariants {
 // deal with the messy conflicts out there
 // Ignore the scalaCompilerPlugin configurations since it is immediately resolved to configure the scala compiler tasks
 configurations.matching{ it.name.contains('CompilerPlugin') == false }.all { Configuration conf ->
-    conf.resolutionStrategy {
-        eachDependency { details ->
-            // change all javax.servlet artifacts to the one used by Spark otherwise these will lead to
-            // SecurityException (signer information wrong)
-            if (details.requested.name.contains("servlet") && !details.requested.name.contains("guice")) {
-                details.useTarget group: "org.eclipse.jetty.orbit", name: "javax.servlet", version: "3.0.0.v201112011016"
-            }
-        }
-    }
-    conf.exclude group: "org.mortbay.jetty"
+//    conf.resolutionStrategy {
+//        eachDependency { details ->
+//            // change all javax.servlet artifacts to the one used by Spark otherwise these will lead to
+//            // SecurityException (signer information wrong)
+//            if (details.requested.name.contains("servlet") && !details.requested.name.contains("guice")) {
+//                details.useTarget group: "org.eclipse.jetty.orbit", name: "javax.servlet", version: "3.0.0.v201112011016"
+//            }
+//        }
+//    }
+//    conf.exclude group: "org.mortbay.jetty"
 }
 
 tasks.withType(ScalaCompile) { ScalaCompile task ->

spark/sql-30/licenses/spark-catalyst_2.12-3.2.0.jar.sha1

@@ -0,0 +1 @@
+6d142b9edf57e3a6dddd26df7e7ff2b3dc860802

spark/sql-30/licenses/spark-core_2.12-3.2.0.jar.sha1

@@ -0,0 +1 @@
+4202b5b58063aacb54fa37a876df18472a980267

spark/sql-30/licenses/spark-sql_2.12-3.2.0.jar.sha1

@@ -0,0 +1 @@
+0a366219179e0354379d8c965edabc9666b99e09

spark/sql-30/licenses/spark-streaming_2.12-3.2.0.jar.sha1

@@ -0,0 +1 @@
+deae68c8374bdf4d7da6943d3fd3a39c8fd04dcf

spark/sql-30/licenses/spark-yarn_2.12-3.2.0.jar.sha1

@@ -0,0 +1 @@
+34a823614c0d46d534c3995cb5c94348c1377c70

spark/sql-30/src/main/scala/org/elasticsearch/spark/sql/streaming/EsSinkMetadataLog.scala

@@ -36,7 +36,14 @@ class EsSinkMetadataLog(settings: Settings, sparkSession: SparkSession, path: St
 
   override protected def defaultCompactInterval: Int = SparkSqlStreamingConfigs.getDefaultCompactInterval(settings)
 
-  override def compactLogs(logs: Seq[EsSinkStatus]): Seq[EsSinkStatus] = logs
+  /**
+   * Compatibility requirement: Spark 3.0 and below used this method instead of shouldRetain.
+   *
+   * This method was renamed in 3.1 but can still be used in 3.0 as long as it is present at runtime.
+   */
+  def compactLogs(logs: Seq[EsSinkStatus]): Seq[EsSinkStatus] = logs
+
+  override def shouldRetain(log: EsSinkStatus, currentTime: Long): Boolean = true
 }
 
 /**