feat: Add Google Drive storage backend with custom endpoint support (#613)

* feat: Add Google Drive storage backend with custom endpoint support

- Implement Google Drive storage backend using service account authentication
- Add support for custom Google Drive API endpoints for testing
- Include comprehensive test setup with mock services
- Add configuration options for Google Drive credentials and folder ID
- Update documentation with Google Drive configuration examples

* remove debug messages

* Improve tests

* Add mounting tip on documentation

* Replace deprecated lib usage

* Fix identation

* Fix googledrive tests

* Fix googledrive support for _FILE credentials

* Remove googledrive test unecessary echos

test/googledrive/credentials.json

@@ -0,0 +1,12 @@
+{
+  "type": "service_account",
+  "project_id": "dummy-project",
+  "private_key_id": "dummykeyid",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCus0CDXvrHhl6a\nLBj7onfU3vRExQQAPstSovS4x3/3BLJNbdMUjrxWnmV5I+Y/U1iw18+8I87CMJDA\n+rIG37tSQ6WYhj2d9ym31O2EgVDQJMkVack/rdXCoWYWn6o7dZcv4K5MEtwW8uWQ\n5PEw0wbK7NIHSSotB9RajzHnLFkSu2XcEThlOp+wkfpTCYGg6+uCBJcMwUBR45eJ\nBLcvifBJVpWaAdj7DcYqWSxRQxensqB5wzCTatwwxDZo3KxnXsf2XRU+C3B71e5q\nb26XTkuIe9W04pj9Fp3fM7RgPSJpElMRFnPUliRhkyppspfYJBYQlpdzDdqKGkGK\nLMDu2c8DAgMBAAECggEAARG8QQ+HJqWNF4VSKCXPO0+C8RtD/IULCNX3NhJzTO4c\nI3ezrp9mlGsUWvPAPAarHmYbgBJtU2I+EZsmse4TaWhcIyVnMm+Dpy1ECucpZoeU\nqIgWe90iW9daBiC3NtRXIlSQNVGjM0mpX8olZM924am6o5/wNh2CP+hsRayBAkqf\nZojppQxYnI+WNNqOlke0T8FoWWm1ZX1gHAJQAeiLpDG675lckP5WxK0RmmKOW/UM\nFU/D4+csMG3eJPhT/Qm3LyAB+pNGpfzHuQXD5jubUhUq2uSsH4ko23wSl0nGHXRW\nX3YhlMDbK4bZtG7YNHQTmh05l6HvEQVbxgHTQLN9gQKBgQDTDDlBQEkLLCWyjmja\nTNt6308CZWZIrWMVtlrpY7S0a6NKm0YGhnXsDGRY4UCNqfMv7xmIw0efN4x90JoX\nglOVeODWgCJHqt6Zzsl8zbEOgbBEvcUO0dMa5PdpMzqd2Y2WghDH1PcrXueMVNXO\nUdf7Rs157LXx5+NouzfGZVmBwQKBgQDT6RwjWV04cxXsCg3QJ06q6YsVeoAawtQE\nWLQ13e0Soa2sBH5TbuOkEQIXVRAVeGSlPfL7N5FsSiZz+ozIhRdTTgNAHqF/TJCf\nEuLEb32Sfw/krLon0LoHBf6GgP+lWqvG4K2YCoAJwBlyHKoQuvbxGer7quuQ29V1\nDqmRL8g5wwKBgQDC0UjU/BOxVYpi/mS6BzKfhR35F0NJGY0a0N+xDBIWbjopN5Z3\nlY2rXXEQPraJTvWnLO8EOUeXKP7ucS6dPvgLRa8/Mr7yK0Aa+TEznOixfHQLsKYE\nXRqje/MLUHfumJHD+sKkxOl5Rr015GYNc62NTjmFMEZwTN+2oQQGhy4NwQKBgBrA\n6W6FD8Hatb/RHSFUdRga2BZkGtxGEKJj2IycchvSEa0P/CroaxEBnLP5Z0hupLY/\n9fdFcrSrP+OQlEmUk/dOeBaWR2lc7z1GEx8dvErMg+Mo82+naHUOiq3Mh3oG0n0P\nTJtPaA7TE+NWPxpRoG+cCBCx6X+mYXKf4USVNcAlAoGBAMH2a8qlnU/lrXSNGcrd\na2TNVi2qDfy0fU6IVFGEydmLMB3wuUUCUcBS6n1d62FqdJY9Rf1wKVIeZgtqJbCv\nOculz64WaXP8TSVrXnqfW8rUsYSTIdV+/P8gxJ9gYGS8E8KZSW5a8yRDc0jcKGI6\nzUJ8tz0Q5jEWC4MdDm7G1XrG\n-----END PRIVATE KEY-----\n",
+  "client_email": "dummy@dummy-project.iam.gserviceaccount.com",
+  "client_id": "dummyclientid",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://oauth2.googleapis.com/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/dummy%40dummy-project.iam.gserviceaccount.com"
+}

test/googledrive/docker-compose.yml

@@ -0,0 +1,52 @@
+services:
+  openapi_mock:
+    image: muonsoft/openapi-mock:0.3.9
+    environment:
+      OPENAPI_MOCK_USE_EXAMPLES: if_present
+      OPENAPI_MOCK_SPECIFICATION_URL: '/etc/openapi/googledrive_v3.yaml'
+    ports:
+      - 8080:8080
+    volumes:
+      - ${SPEC_FILE:-./googledrive_v3.yaml}:/etc/openapi/googledrive_v3.yaml
+
+  oauth2_mock:
+    image: ghcr.io/navikt/mock-oauth2-server:1.0.0
+    ports:
+      - 8090:8090
+    environment:
+      PORT: 8090
+      JSON_CONFIG_PATH: '/etc/oauth2/config.json'
+    volumes:
+      - ./oauth2_config.json:/etc/oauth2/config.json
+
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    hostname: hostnametoken
+    depends_on:
+      - openapi_mock
+      - oauth2_mock
+    restart: always
+    environment:
+      BACKUP_FILENAME_EXPAND: 'true'
+      BACKUP_FILENAME: test-$$HOSTNAME.tar.gz
+      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
+      BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
+      BACKUP_PRUNING_LEEWAY: 5s
+      BACKUP_PRUNING_PREFIX: test
+      GOOGLE_DRIVE_ENDPOINT: http://openapi_mock:8080
+      GOOGLE_DRIVE_TOKEN_URL: http://oauth2_mock:8090/issuer1/token
+      GOOGLE_DRIVE_CREDENTIALS_JSON_FILE: /etc/gdrive/credentials.json
+      GOOGLE_DRIVE_FOLDER_ID: "root"
+    volumes:
+      - app_data:/backup/app_data:ro
+      - ./credentials.json:/etc/gdrive/credentials.json
+
+  offen:
+    image: offen/offen:latest
+    labels:
+      - docker-volume-backup.stop-during-backup=true
+    volumes:
+      - app_data:/var/opt/offen
+
+volumes:
+  app_data:

test/googledrive/googledrive_v3.yaml

@@ -0,0 +1,139 @@
+openapi: 3.0.1
+info:
+  title: Minimal Google Drive API Mock
+  version: 1.0.0
+  description: Minimal mock implementation of Google Drive API v3 for testing
+servers:
+  - url: /
+paths:
+  /upload/drive/v3/files:
+    post:
+      summary: Upload file to Google Drive
+      parameters:
+        - name: uploadType
+          in: query
+          schema:
+            type: string
+        - name: fields
+          in: query
+          schema:
+            type: string
+        - name: supportsAllDrives
+          in: query
+          schema:
+            type: boolean
+        - name: alt
+          in: query
+          schema:
+            type: string
+        - name: prettyPrint
+          in: query
+          schema:
+            type: boolean
+      requestBody:
+        content:
+          multipart/related:
+            schema:
+              type: string
+              format: binary
+      responses:
+        '200':
+          description: File uploaded successfully
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  id:
+                    type: string
+                    description: "The ID of the file"
+                  name:
+                    type: string
+                    description: "The name of the file (extracted from request.metadata.name)"
+                  mimeType:
+                    type: string
+                    description: "The MIME type of the file"
+                  size:
+                    type: string
+                    description: "The size of the file in bytes"
+              examples:
+                UploadSuccess:
+                  summary: "Response when file is uploaded successfully"
+                  description: "The response includes the filename from the request metadata"
+                  value:
+                    id: "1BxiMVs0XRA5nFMdKvBdBZjgmUUqptlbs74OgvE2upms"
+                    name: "test-backup.tar.gz"
+                    mimeType: "application/gzip"
+  /files:
+    get:
+      summary: List files in Google Drive
+      parameters:
+        - name: q
+          in: query
+          schema:
+            type: string
+            description: "A query for filtering the file results"
+        - name: fields
+          in: query
+          schema:
+            type: string
+        - name: supportsAllDrives
+          in: query
+          schema:
+            type: boolean
+        - name: includeItemsFromAllDrives
+          in: query
+          schema:
+            type: boolean
+      responses:
+        '200':
+          description: Files listed successfully
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  files:
+                    type: array
+                    items:
+                      type: object
+                      properties:
+                        id:
+                          type: string
+                          description: "The ID of the file"
+                        name:
+                          type: string
+                          description: "The name of the file"
+                        mimeType:
+                          type: string
+                          description: "The MIME type of the file"
+                        createdTime:
+                          type: string
+                          description: "The time the file was created"
+              examples:
+                FilesList:
+                  value:
+                    files:
+                      - id: "1BxiMVs0XRA5nFMdKvBdBZjgmUUqptlbs74OgvE2upms"
+                        name: "test-hostnametoken.tar.gz"
+                        createdTime: "CREATED_TIME_1"
+                      - id: "jgmUUqptlbs74OgvE2upms1BxiMVs0XRA5nFMdKvBdBZ"
+                        name: "test-hostnametoken-old.tar.gz"
+                        createdTime: "CREATED_TIME_2"
+
+  /files/{fileId}:
+    delete:
+      summary: Delete a file from Google Drive
+      parameters:
+        - name: fileId
+          in: path
+          required: true
+          schema:
+            type: string
+        - name: supportsAllDrives
+          in: query
+          schema:
+            type: boolean
+      responses:
+        '204':
+          description: File deleted successfully

test/googledrive/oauth2_config.json

@@ -0,0 +1,37 @@
+{
+   "interactiveLogin": true,
+   "httpServer": "NettyWrapper",
+   "tokenCallbacks": [
+       {
+           "issuerId": "issuer1",
+           "tokenExpiry": 120,
+           "requestMappings": [
+               {
+                   "requestParam": "scope",
+                   "match": "scope1",
+                   "claims": {
+                       "sub": "subByScope",
+                       "aud": [
+                           "audByScope"
+                       ]
+                   }
+               }
+           ]
+       },
+       {
+           "issuerId": "issuer2",
+           "requestMappings": [
+               {
+                   "requestParam": "someparam",
+                   "match": "somevalue",
+                   "claims": {
+                       "sub": "subBySomeParam",
+                       "aud": [
+                           "audBySomeParam"
+                       ]
+                   }
+               }
+           ]
+       }
+   ]
+}

test/googledrive/run.sh

@@ -0,0 +1,59 @@
+#!/bin/sh
+
+set -e
+
+cd "$(dirname "$0")"
+. ../util.sh
+current_test=$(basename $(pwd))
+
+export SPEC_FILE=$(mktemp -d)/googledrive_v3.yaml
+cp googledrive_v3.yaml $SPEC_FILE
+sed -i 's/CREATED_TIME_1/'"$(date "+%Y-%m-%dT%H:%M:%SZ")/g" $SPEC_FILE
+sed -i 's/CREATED_TIME_2/'"$(date "+%Y-%m-%dT%H:%M:%SZ" -d "14 days ago")/g" $SPEC_FILE
+
+docker compose up -d --quiet-pull
+sleep 5
+
+logs=$(docker compose exec backup backup | tee /dev/stderr)
+
+sleep 5
+
+expect_running_containers "4"
+
+if echo "$logs" | grep -q "ERROR"; then
+  fail "Backup failed, check logs for error"
+else
+  pass "Backup succeeded, no errors reported."
+fi
+
+# The second part of this test checks if backups get deleted when the retention
+# is set to 0 days (which it should not as it would mean all backups get deleted)
+BACKUP_RETENTION_DAYS="0" docker compose up -d
+sleep 5
+
+logs=$(docker compose exec -T backup backup | tee /dev/stderr)
+
+if echo "$logs" | grep -q "Refusing to do so, please check your configuration"; then
+  pass "Remote backups have not been deleted."
+else
+  fail "Remote backups would have been deleted: $logs"
+fi
+
+# The third part of this test checks if old backups get deleted when the retention
+# is set to 7 days (which it should)
+BACKUP_RETENTION_DAYS="7" docker compose up -d
+sleep 5
+
+info "Create second backup and prune"
+
+logs=$(docker compose exec -T backup backup | tee /dev/stderr)
+
+if echo "$logs" | grep -q "Pruned 1 out of 2 backups as they were older"; then
+  pass "Old remote backup has been pruned, new one is still present."
+elif echo "$logs" | grep -q "ERROR"; then
+  fail "Pruning failed, errors reported: $logs"
+elif echo "$logs" | grep -q "None of 1 existing backups were pruned"; then
+  fail "Pruning failed, old backup has not been pruned: $logs"
+else
+  fail "Pruning failed, unknown result: $logs"
+fi