diff --git a/src/main/scala/com/github/riskidentdms/spark/google/spreadsheets/Credentials.scala b/src/main/scala/com/github/riskidentdms/spark/google/spreadsheets/Credentials.scala new file mode 100644 index 0000000..de79b79 --- /dev/null +++ b/src/main/scala/com/github/riskidentdms/spark/google/spreadsheets/Credentials.scala @@ -0,0 +1,48 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.riskidentdms.spark.google.spreadsheets + +import com.google.api.services.sheets.v4.SheetsScopes +import com.google.auth.oauth2.{GoogleCredentials, OAuth2Credentials} + +import java.time.Duration +import scala.collection.JavaConverters.seqAsJavaListConverter +import scala.io.Source + +object Credentials { + private val scopes = List(SheetsScopes.SPREADSHEETS) + + def credentialsFromFile(file: String): OAuth2Credentials = { + val lines = Source.fromFile(file) + try { + credentialsFromJsonString(lines.getLines().mkString) + } finally { + lines.close() + } + } + + def credentialsFromJsonString(oauth2JSON: String): OAuth2Credentials = { + val credentials: GoogleCredentials = GoogleCredentials.fromStream( + new java.io.ByteArrayInputStream(oauth2JSON.getBytes(java.nio.charset.StandardCharsets.UTF_8)) + ).createScoped(scopes.asJava) + + credentials.refreshIfExpired() + val accessToken = credentials.refreshAccessToken() + + GoogleCredentials.newBuilder() + .setAccessToken(accessToken) + .setRefreshMargin(Duration.ofDays(1)) + .build() + } +} diff --git a/src/main/scala/com/github/riskidentdms/spark/google/spreadsheets/DefaultSource.scala b/src/main/scala/com/github/riskidentdms/spark/google/spreadsheets/DefaultSource.scala index a061125..774ce06 100644 --- a/src/main/scala/com/github/riskidentdms/spark/google/spreadsheets/DefaultSource.scala +++ b/src/main/scala/com/github/riskidentdms/spark/google/spreadsheets/DefaultSource.scala @@ -13,7 +13,6 @@ */ package com.github.riskidentdms.spark.google.spreadsheets -import com.github.riskidentdms.spark.google.spreadsheets.util.Credentials import org.apache.spark.sql.sources.{BaseRelation, CreatableRelationProvider, RelationProvider, SchemaRelationProvider} import org.apache.spark.sql.types.StructType import org.apache.spark.sql.{DataFrame, SQLContext, SaveMode} diff --git a/src/main/scala/com/github/riskidentdms/spark/google/spreadsheets/SparkSpreadsheetService.scala b/src/main/scala/com/github/riskidentdms/spark/google/spreadsheets/SparkSpreadsheetService.scala index 192827c..9d069ec 100644 --- a/src/main/scala/com/github/riskidentdms/spark/google/spreadsheets/SparkSpreadsheetService.scala +++ b/src/main/scala/com/github/riskidentdms/spark/google/spreadsheets/SparkSpreadsheetService.scala @@ -23,12 +23,13 @@ import java.util.{List => JavaList} import scala.collection.JavaConverters._ import com.google.api.client.json.gson.GsonFactory import com.google.auth.http.HttpCredentialsAdapter +import com.google.auth.oauth2.OAuth2Credentials import scala.Option.option2Iterable import scala.util.Try object SparkSpreadsheetService { - private val APP_NAME = "spark-google-spreadsheets-1.0.0" + private val APP_NAME = "spark-google-spreadsheets" private val HTTP_TRANSPORT: NetHttpTransport = GoogleNetHttpTransport.newTrustedTransport() private val JSON_FACTORY: GsonFactory = GsonFactory.getDefaultInstance @@ -94,10 +95,10 @@ object SparkSpreadsheetService { } } - case class SparkSpreadsheetContext(credentials: HttpCredentialsAdapter) { + case class SparkSpreadsheetContext(credentials: OAuth2Credentials) { lazy val service: Sheets = - new Sheets.Builder(HTTP_TRANSPORT, JSON_FACTORY, credentials) + new Sheets.Builder(HTTP_TRANSPORT, JSON_FACTORY, new HttpCredentialsAdapter(credentials)) .setApplicationName(APP_NAME) .build() @@ -267,7 +268,7 @@ object SparkSpreadsheetService { * @param credentials * @return */ - def apply(credentials: HttpCredentialsAdapter): SparkSpreadsheetContext = + def apply(credentials: OAuth2Credentials): SparkSpreadsheetContext = SparkSpreadsheetContext(credentials) /** diff --git a/src/main/scala/com/github/riskidentdms/spark/google/spreadsheets/Util.scala b/src/main/scala/com/github/riskidentdms/spark/google/spreadsheets/Util.scala index a1345ee..00a12a1 100644 --- a/src/main/scala/com/github/riskidentdms/spark/google/spreadsheets/Util.scala +++ b/src/main/scala/com/github/riskidentdms/spark/google/spreadsheets/Util.scala @@ -1,3 +1,16 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.github.riskidentdms.spark.google.spreadsheets import com.google.api.services.sheets.v4.model.{CellData, ExtendedValue, RowData} diff --git a/src/main/scala/com/github/riskidentdms/spark/google/spreadsheets/util/Credentials.scala b/src/main/scala/com/github/riskidentdms/spark/google/spreadsheets/util/Credentials.scala deleted file mode 100644 index 03c4a03..0000000 --- a/src/main/scala/com/github/riskidentdms/spark/google/spreadsheets/util/Credentials.scala +++ /dev/null @@ -1,37 +0,0 @@ -package com.github.riskidentdms.spark.google.spreadsheets.util - -import com.google.api.services.sheets.v4.SheetsScopes -import com.google.auth.http.HttpCredentialsAdapter -import com.google.auth.oauth2.GoogleCredentials - -import java.time.Duration -import scala.collection.JavaConverters.seqAsJavaListConverter -import scala.io.Source - -object Credentials { - private val scopes = List(SheetsScopes.SPREADSHEETS) - - def credentialsFromFile(filename: String): HttpCredentialsAdapter = { - val lines = Source.fromFile(filename) - try { - credentialsFromJsonString(lines.getLines().mkString) - } finally { - lines.close() - } - } - - def credentialsFromJsonString(oauth2JSON: String): HttpCredentialsAdapter = { - val credentials: GoogleCredentials = GoogleCredentials.fromStream( - new java.io.ByteArrayInputStream(oauth2JSON.getBytes(java.nio.charset.StandardCharsets.UTF_8)) - ).createScoped(scopes.asJava) - - credentials.refreshIfExpired() - val accessToken = credentials.refreshAccessToken() - - val oAuth2Credentials = GoogleCredentials.newBuilder() - .setAccessToken(accessToken) - .setRefreshMargin(Duration.ofDays(1)) - .build() - new HttpCredentialsAdapter(oAuth2Credentials) - } -} diff --git a/src/main/scala/com/github/riskidentdms/spark/google/spreadsheets/util/TypeCast.scala b/src/main/scala/com/github/riskidentdms/spark/google/spreadsheets/util/TypeCast.scala index 374a1d9..bef9232 100644 --- a/src/main/scala/com/github/riskidentdms/spark/google/spreadsheets/util/TypeCast.scala +++ b/src/main/scala/com/github/riskidentdms/spark/google/spreadsheets/util/TypeCast.scala @@ -1,3 +1,17 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.github.riskidentdms.spark.google.spreadsheets.util import java.math.BigDecimal diff --git a/src/test/scala/com/github/riskidentdms/spark/google/spreadsheets/SparkSpreadsheetServiceReadSuite.scala b/src/test/scala/com/github/riskidentdms/spark/google/spreadsheets/SparkSpreadsheetServiceReadSuite.scala index 0ed6de2..78ac3f2 100644 --- a/src/test/scala/com/github/riskidentdms/spark/google/spreadsheets/SparkSpreadsheetServiceReadSuite.scala +++ b/src/test/scala/com/github/riskidentdms/spark/google/spreadsheets/SparkSpreadsheetServiceReadSuite.scala @@ -13,8 +13,6 @@ */ package com.github.riskidentdms.spark.google.spreadsheets -import com.github.riskidentdms.spark.google.spreadsheets.util.Credentials - import org.scalatest.BeforeAndAfter import org.scalatest.flatspec.AnyFlatSpec diff --git a/src/test/scala/com/github/riskidentdms/spark/google/spreadsheets/SparkSpreadsheetServiceWriteSuite.scala b/src/test/scala/com/github/riskidentdms/spark/google/spreadsheets/SparkSpreadsheetServiceWriteSuite.scala index d1bb0b6..01016fd 100644 --- a/src/test/scala/com/github/riskidentdms/spark/google/spreadsheets/SparkSpreadsheetServiceWriteSuite.scala +++ b/src/test/scala/com/github/riskidentdms/spark/google/spreadsheets/SparkSpreadsheetServiceWriteSuite.scala @@ -14,7 +14,6 @@ package com.github.riskidentdms.spark.google.spreadsheets import SparkSpreadsheetService.SparkSpreadsheet -import com.github.riskidentdms.spark.google.spreadsheets.util.Credentials import com.github.riskidentdms.spark.google.spreadsheets.SparkSpreadsheetService.SparkSpreadsheet import com.google.api.services.sheets.v4.model.{CellData, ExtendedValue, RowData} import org.apache.spark.sql.types.{DataTypes, StructField, StructType} diff --git a/src/test/scala/com/github/riskidentdms/spark/google/spreadsheets/SpreadsheetSuite.scala b/src/test/scala/com/github/riskidentdms/spark/google/spreadsheets/SpreadsheetSuite.scala index 5e56076..9b6f07d 100644 --- a/src/test/scala/com/github/riskidentdms/spark/google/spreadsheets/SpreadsheetSuite.scala +++ b/src/test/scala/com/github/riskidentdms/spark/google/spreadsheets/SpreadsheetSuite.scala @@ -14,7 +14,6 @@ package com.github.riskidentdms.spark.google.spreadsheets import SparkSpreadsheetService.SparkSpreadsheetContext -import com.github.riskidentdms.spark.google.spreadsheets.util.Credentials import org.apache.spark.rdd.RDD import org.apache.spark.sql.types._ import org.apache.spark.sql.{DataFrame, Row, SQLContext, SparkSession} @@ -107,9 +106,9 @@ class SpreadsheetSuite extends AnyFlatSpec with BeforeAndAfter { trait PersonData { val personsSchema = StructType(List( - StructField("id", IntegerType, true), - StructField("firstname", StringType, true), - StructField("lastname", StringType, true))) + StructField("id", IntegerType, nullable = true), + StructField("firstname", StringType, nullable = true), + StructField("lastname", StringType, nullable = true))) } trait PersonDataFrame extends PersonData { @@ -139,7 +138,6 @@ class SpreadsheetSuite extends AnyFlatSpec with BeforeAndAfter { behavior of "A DataFrame" it should "be saved as a sheet" in new PersonDataFrame { - import com.github.riskidentdms.spark.google.spreadsheets._ withEmptyWorksheet { workSheetName => personsDF.write .option("credentialsJson", oAuthJson) @@ -238,7 +236,7 @@ class SpreadsheetSuite extends AnyFlatSpec with BeforeAndAfter { } trait UnderscoreDataFrame { - val aSchema: StructType = StructType(List(StructField("foo_bar", IntegerType, true))) + val aSchema: StructType = StructType(List(StructField("foo_bar", IntegerType, nullable = true))) val aRows = Seq(Row(1), Row(2), Row(3)) val aRDD: RDD[Row] = sqlContext.sparkContext.parallelize(aRows) val aDF: DataFrame = sqlContext.createDataFrame(aRDD, aSchema)