Repository: gisaia/aias
Pull request #272: Feat/push
Author: login: WilliGautier
Created: 2025-05-20T07:41:44Z | Merged: 2025-05-21T17:05:39Z

Changes in aias-common:

- Add push function to  AccessManager storage (local, gs)
- Fix gs authentication configuration
- Fix python dependencies (missing requirements.txt and in setup.py) 


Fix https://github.com/gisaia/aias/issues/273


Diff:
diff --git a/python/aias_common/access/configuration.py b/python/aias_common/access/configuration.py
index b8c60394..7eb0ec3b 100644
--- a/python/aias_common/access/configuration.py
+++ b/python/aias_common/access/configuration.py
@@ -27,28 +27,25 @@ class GoogleStorageConstants(str, enum.Enum):
     AUTH_URI = "https://accounts.google.com/o/oauth2/auth"
     TOKEN_URI = "https://oauth2.googleapis.com/token"
     AUTH_PROVIDER_CERT_URL = "https://www.googleapis.com/oauth2/v1/certs"
+    UNIVERSE_DOMAIN = "googleapis.com"
 
 
-class GoogleStorageApiKey(StorageConfiguration):
+class GoogleStorageApiKey(BaseModel):
     type: Literal["service_account"] = "service_account"
     project_id: str
     private_key_id: str
     private_key: str
+    client_email: str
     client_id: str | None = Field(None)
-    auth_uri: Literal[GoogleStorageConstants.AUTH_URI] = GoogleStorageConstants.AUTH_URI
-    token_uri: Literal[GoogleStorageConstants.TOKEN_URI] = GoogleStorageConstants.TOKEN_URI
-    auth_provider_x509_cert_url: Literal[GoogleStorageConstants.AUTH_PROVIDER_CERT_URL] = GoogleStorageConstants.AUTH_PROVIDER_CERT_URL
-    universe_domain: Literal["googleapis.com"] = "googleapis.com"
+    auth_uri: Literal[GoogleStorageConstants.AUTH_URI] = GoogleStorageConstants.AUTH_URI.value
+    token_uri: Literal[GoogleStorageConstants.TOKEN_URI] = GoogleStorageConstants.TOKEN_URI.value
+    auth_provider_x509_cert_url: Literal[GoogleStorageConstants.AUTH_PROVIDER_CERT_URL] = GoogleStorageConstants.AUTH_PROVIDER_CERT_URL.value
+    universe_domain: Literal[GoogleStorageConstants.UNIVERSE_DOMAIN] = GoogleStorageConstants.UNIVERSE_DOMAIN.value
 
     @computed_field
     @property
     def client_x509_cert_url(self) -> str:
-        return f"https://www.googleapis.com/robot/v1/metadata/x509/{self.project_id}%40appspot.gserviceaccount.com"
-
-    @computed_field
-    @property
-    def client_email(self) -> str:
-        return f"{self.project_id}@appspot.gserviceaccount.com"
+        return f"https://www.googleapis.com/robot/v1/metadata/x509/{self.client_email.replace('@', '%40')}"
 
 
 class GoogleStorageConfiguration(StorageConfiguration):
diff --git a/python/aias_common/access/manager.py b/python/aias_common/access/manager.py
index 7de938d4..0e050378 100644
--- a/python/aias_common/access/manager.py
+++ b/python/aias_common/access/manager.py
@@ -77,9 +77,31 @@ def check_local_path_writable(href: str):
         """
         Checks that the path is a writable path for at least one of the file storages
         """
-        is_authorized = any(map(lambda s: s.is_path_authorized(href, AccessType.WRITE), filter(lambda s: s.storage_configuration.type == "file", AccessManager.storages)))
+        is_authorized = any(map(lambda s: s.is_path_authorized(href, AccessType.WRITE),
+                                filter(lambda s: s.storage_configuration.type == "file", AccessManager.storages)))
         if not is_authorized:
-            raise ValueError("Destination path is not authorized")
+            raise ValueError(f"Local path '{href}' is not writable")
+
+    @staticmethod
+    def check_local_path_readable(href: str):
+        """
+        Checks that the path is a readable path for at least one of the file storages
+        """
+        is_authorized = any(map(lambda s: s.is_path_authorized(href, AccessType.READ),
+                                filter(lambda s: s.storage_configuration.type == "file", AccessManager.storages)))
+        if not is_authorized:
+            raise ValueError(f"Local path '{href}' is not readable")
+
+    @staticmethod
+    def push(href: str, dst: str):
+        """
+        Push a file from a local storage to write it in a storage.
+        If the destination storage is local, then it is a copy. Otherwise it is an upload.
+        """
+        storage = AccessManager.resolve_storage(dst)
+        AccessManager.check_local_path_readable(href)
+
+        storage.push(href, dst)
 
     @staticmethod
     def pull(href: str, dst: str):
diff --git a/python/aias_common/access/storages/abstract.py b/python/aias_common/access/storages/abstract.py
index 458dbe57..da765914 100644
--- a/python/aias_common/access/storages/abstract.py
+++ b/python/aias_common/access/storages/abstract.py
@@ -19,7 +19,7 @@ def get_configuration(self) -> AnyStorageConfiguration:
         Returns:
             AnyStorageConfiguration: storage configuration
         """
-    ...
+        ...
 
     @abstractmethod
     def get_storage_parameters(self) -> dict:
@@ -79,6 +79,19 @@ def pull(self, href: str, dst: str):
         if scheme != "" and scheme != "file":
             raise ValueError("Destination must be on the local filesystem")
 
+    @abstractmethod
+    def push(self, href: str, dst: str):
+        """Copy/upload the desired file from local to write it on the file system
+
+        Args:
+            href (str): File to upload
+            dst (str): Destination of the file
+        """
+        # Check that href is local
+        scheme = urlparse(href).scheme
+        if scheme != "" and scheme != "file":
+            raise ValueError("Source file to upload must be on the local filesystem")
+
     @abstractmethod
     def is_file(self, href: str) -> bool:
         """Returns whether the specified href is a file
diff --git a/python/aias_common/access/storages/file.py b/python/aias_common/access/storages/file.py
index 157849be..bb66e170 100644
--- a/python/aias_common/access/storages/file.py
+++ b/python/aias_common/access/storages/file.py
@@ -44,6 +44,14 @@ def pull(self, href: str, dst: str):
 
         shutil.copy(href, dst)
 
+    def push(self, href: str, dst: str):
+        super().push(href, dst)
+
+        if not self.is_path_authorized(dst, AccessType.WRITE):
+            raise ValueError('The desired output path is not authorized')
+
+        shutil.copy(href, dst)
+
     def is_file(self, href: str):
         return os.path.isfile(href)
 
diff --git a/python/aias_common/access/storages/gs.py b/python/aias_common/access/storages/gs.py
index 3c1870fe..46c51560 100644
--- a/python/aias_common/access/storages/gs.py
+++ b/python/aias_common/access/storages/gs.py
@@ -19,8 +19,8 @@ def get_storage_parameters(self):
         if self.get_configuration().is_anon_client:
             client = Client.create_anonymous_client()
         else:
-            credentials = service_account.Credentials.from_service_account_info(self.get_configuration().api_key)
-            client = Client("APROC", credentials=credentials)
+            credentials = service_account.Credentials.from_service_account_info(self.get_configuration().api_key.model_dump())
+            client = Client(project=self.get_configuration().api_key.project_id, credentials=credentials)
 
         return {"client": client}
 
@@ -41,7 +41,11 @@ def __get_bucket(self):
 
     def __get_blob(self, href: str):
         bucket = self.__get_bucket()
-        return bucket.get_blob(urlparse(href).path[1:] or "/")
+        return bucket.get_blob(urlparse(href).path.lstrip("/") or "/")
+
+    def __create_blob(self, href: str):
+        bucket = self.__get_bucket()
+        return bucket.blob(urlparse(href).path.lstrip("/") or "/")
 
     def exists(self, href: str):
         return self.is_file(href) or self.is_dir(href)
@@ -69,6 +73,15 @@ def pull(self, href: str, dst: str):
 
         blob.download_to_filename(dst)
 
+    def push(self, href: str, dst: str):
+        super().push(href, dst)
+
+        blob = self.__create_blob(dst)
+        if blob is None:
+            raise LookupError(f"Can't create blob: {dst}")
+
+        blob.upload_from_filename(href)
+
     @ttl_lru_cache(ttl=AbstractStorage.cache_tt, max_size=1024)
     def __list_blobs(self, source: str) -> list[File]:
         """
diff --git a/python/aias_common/access/storages/http.py b/python/aias_common/access/storages/http.py
index f172ec7c..8a7c5828 100644
--- a/python/aias_common/access/storages/http.py
+++ b/python/aias_common/access/storages/http.py
@@ -34,6 +34,10 @@ def pull(self, href: str, dst: str):
         super().pull(href, dst)
         requests_get(href, dst, self.get_configuration().headers)
 
+    def push(self, href: str, dst: str):
+        super().push(href, dst)
+        raise NotImplementedError("'push' method is not available for http storage")
+
     def is_file(self, href: str):
         return self.exists(href)
 
diff --git a/python/aias_common/access/storages/s3.py b/python/aias_common/access/storages/s3.py
index 86bab7c0..9abd3eea 100644
--- a/python/aias_common/access/storages/s3.py
+++ b/python/aias_common/access/storages/s3.py
@@ -90,6 +90,10 @@ def pull(self, href: str, dst: str):
             for chunk in obj['Body'].iter_chunks(50 * 1024):
                 f.write(chunk)
 
+    def push(self, href: str, dst: str):
+        super().push(href, dst)
+        raise NotImplementedError("'push' method has not been implemented yet for s3 storage")
+
     def __head_object(self, href: str):
         return self.get_storage_parameters()["client"].head_object(
                 Bucket=self.get_configuration().bucket,
diff --git a/python/aias_common/requirements.txt b/python/aias_common/requirements.txt
new file mode 100644
index 00000000..90e0121c
--- /dev/null
+++ b/python/aias_common/requirements.txt
@@ -0,0 +1,8 @@
+setuptools==80.4.0
+fastapi_utilities==0.3.1
+ecs_logging==2.2.0
+google-cloud-storage==2.5.0
+pydantic==2.10.6
+requests==2.31.0
+smart_open==6.2.0
+boto3==1.24.89
diff --git a/release/materials/aias_common/setup.py b/release/materials/aias_common/setup.py
index 1d607d57..215e1ad4 100644
--- a/release/materials/aias_common/setup.py
+++ b/release/materials/aias_common/setup.py
@@ -13,5 +13,6 @@
     packages=setuptools.find_packages(where="src"),
     python_requires='>=3.10',
     package_dir={'': 'src'},
-    install_requires=['ecs_logging']
+    install_requires=['ecs_logging', 'google-cloud-storage==2.5.0', 'pydantic==2.10.6', 'requests==2.31.0',
+                      'smart_open==6.2.0', 'boto3==1.24.89', 'fastapi_utilities']
 )
diff --git a/release/release.sh b/release/release.sh
index 5da4c056..243ad63b 100755
--- a/release/release.sh
+++ b/release/release.sh
@@ -2,12 +2,12 @@
 set -o errexit
 [ -z "$1" ] && echo "Please provide the version" && exit 1;
 VERSION=$1
-echo "Build and releas the image with version ${VERSION}"
+echo "Build and release the image with version ${VERSION}"
 
 send_chat_message(){
     MESSAGE=$1
     if [ -z "$GOOGLE_CHAT_RELEASE_CHANEL" ] ; then
-        echo "Environement variable GOOGLE_CHAT_RELEASE_CHANEL is not definied ... skipping message publishing"
+        echo "Environment variable GOOGLE_CHAT_RELEASE_CHANEL is not defined ... skipping message publishing"
     else
         DATA='{"text":"'${MESSAGE}'"}'
         echo $DATA


Reviews:
user: login: QuCMGisaia
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-05-20T08:51:31Z
user: login: WilliGautier
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-20T09:10:53Z
user: login: WilliGautier
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-20T09:18:49Z
user: login: QuCMGisaia
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-20T09:29:16Z
user: login: WilliGautier
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-20T09:36:24Z
user: login: QuCMGisaia
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-20T10:06:21Z
user: login: sylvaingaudan
state: APPROVED
author_association: MEMBER
submitted_at: 2025-05-21T17:02:45Z

Review comments:
diff_hunk: @@ -29,27 +29,23 @@ class GoogleStorageConstants(str, enum.Enum):
     AUTH_PROVIDER_CERT_URL = "https://www.googleapis.com/oauth2/v1/certs"
 
 
-class GoogleStorageApiKey(StorageConfiguration):
+class GoogleStorageApiKey(BaseModel):
     type: Literal["service_account"] = "service_account"
     project_id: str
     private_key_id: str
     private_key: str
+    client_email: str
     client_id: str | None = Field(None)
-    auth_uri: Literal[GoogleStorageConstants.AUTH_URI] = GoogleStorageConstants.AUTH_URI
-    token_uri: Literal[GoogleStorageConstants.TOKEN_URI] = GoogleStorageConstants.TOKEN_URI
-    auth_provider_x509_cert_url: Literal[GoogleStorageConstants.AUTH_PROVIDER_CERT_URL] = GoogleStorageConstants.AUTH_PROVIDER_CERT_URL
+    auth_uri: Literal[GoogleStorageConstants.AUTH_URI] = GoogleStorageConstants.AUTH_URI.value
+    token_uri: Literal[GoogleStorageConstants.TOKEN_URI] = GoogleStorageConstants.TOKEN_URI.value
+    auth_provider_x509_cert_url: Literal[GoogleStorageConstants.AUTH_PROVIDER_CERT_URL] = GoogleStorageConstants.AUTH_PROVIDER_CERT_URL.value
     universe_domain: Literal["googleapis.com"] = "googleapis.com"
 
     @computed_field
     @property
     def client_x509_cert_url(self) -> str:
         return f"https://www.googleapis.com/robot/v1/metadata/x509/{self.project_id}%40appspot.gserviceaccount.com"
path: python/aias_common/access/configuration.py
user: login: QuCMGisaia
body: this should be modified following the change of how the client_email is computed. It should be `f"https://www.googleapis.com/robot/v1/metadata/x509/{self.client_email.replace('@', '%40')}"` to properly url-ify the email adress
created_at: 2025-05-20T08:46:27Z
author_association: CONTRIBUTOR
path: python/aias_common/requirements.txt
user: login: QuCMGisaia
body: this should replace the file `extensions/requirements.storage.txt`
created_at: 2025-05-20T08:50:13Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,8 @@
+setuptools
+fastapi_utilities
path: python/aias_common/requirements.txt
user: login: QuCMGisaia
body: this should be with a fixed version to make sure that it still works with the fastapi version of the other projects
created_at: 2025-05-20T08:50:46Z
author_association: CONTRIBUTOR
path: python/aias_common/requirements.txt
user: login: WilliGautier
body: Do you want me to remove this file? 
created_at: 2025-05-20T09:10:53Z
author_association: CONTRIBUTOR
diff_hunk: @@ -29,27 +29,23 @@ class GoogleStorageConstants(str, enum.Enum):
     AUTH_PROVIDER_CERT_URL = "https://www.googleapis.com/oauth2/v1/certs"
 
 
-class GoogleStorageApiKey(StorageConfiguration):
+class GoogleStorageApiKey(BaseModel):
     type: Literal["service_account"] = "service_account"
     project_id: str
     private_key_id: str
     private_key: str
+    client_email: str
     client_id: str | None = Field(None)
-    auth_uri: Literal[GoogleStorageConstants.AUTH_URI] = GoogleStorageConstants.AUTH_URI
-    token_uri: Literal[GoogleStorageConstants.TOKEN_URI] = GoogleStorageConstants.TOKEN_URI
-    auth_provider_x509_cert_url: Literal[GoogleStorageConstants.AUTH_PROVIDER_CERT_URL] = GoogleStorageConstants.AUTH_PROVIDER_CERT_URL
+    auth_uri: Literal[GoogleStorageConstants.AUTH_URI] = GoogleStorageConstants.AUTH_URI.value
+    token_uri: Literal[GoogleStorageConstants.TOKEN_URI] = GoogleStorageConstants.TOKEN_URI.value
+    auth_provider_x509_cert_url: Literal[GoogleStorageConstants.AUTH_PROVIDER_CERT_URL] = GoogleStorageConstants.AUTH_PROVIDER_CERT_URL.value
     universe_domain: Literal["googleapis.com"] = "googleapis.com"
 
     @computed_field
     @property
     def client_x509_cert_url(self) -> str:
         return f"https://www.googleapis.com/robot/v1/metadata/x509/{self.project_id}%40appspot.gserviceaccount.com"
path: python/aias_common/access/configuration.py
user: login: WilliGautier
body: Ok, it is strange that it works without this but clearly better to do it !
created_at: 2025-05-20T09:18:49Z
author_association: CONTRIBUTOR
path: python/aias_common/requirements.txt
user: login: QuCMGisaia
body: The file `extensions/requirements.storage.txt` yes, but it is imported in the docker images also, so you would have to modify that also. The goal is to just have to add the aias_common dependencies to the projects that need it, instead of having this duplicate file that I had created before aias_common was created
created_at: 2025-05-20T09:29:16Z
author_association: CONTRIBUTOR
path: python/aias_common/requirements.txt
user: login: WilliGautier
body: I would rather create an issue and do it in another PR to test it properly
created_at: 2025-05-20T09:36:24Z
author_association: CONTRIBUTOR
path: python/aias_common/requirements.txt
user: login: QuCMGisaia
body: Yes it can be done separately, I'll let you create the issue !
created_at: 2025-05-20T10:06:21Z
author_association: CONTRIBUTOR

PR comments:
user: login: sonarqubecloud[bot]
created_at: 2025-05-21T16:39:18Z
body: ## [![Quality Gate Passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/qg-passed-20px.png 'Quality Gate Passed')](https://sonarcloud.io/dashboard?id=gisaia_aias&pullRequest=272) **Quality Gate passed**  
Issues  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0 New issues](https://sonarcloud.io/project/issues?id=gisaia_aias&pullRequest=272&issueStatuses=OPEN,CONFIRMED&sinceLeakPeriod=true)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/accepted-16px.png '') [0 Accepted issues](https://sonarcloud.io/project/issues?id=gisaia_aias&pullRequest=272&issueStatuses=ACCEPTED)

Measures  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=gisaia_aias&pullRequest=272&issueStatuses=OPEN,CONFIRMED&sinceLeakPeriod=true)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0.0% Coverage on New Code](https://sonarcloud.io/component_measures?id=gisaia_aias&pullRequest=272&metric=new_coverage&view=list)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0.0% Duplication on New Code](https://sonarcloud.io/component_measures?id=gisaia_aias&pullRequest=272&metric=new_duplicated_lines_density&view=list)  
  
[See analysis details on SonarQube Cloud](https://sonarcloud.io/dashboard?id=gisaia_aias&pullRequest=272)


author_association: NONE