S3 Compatible Storage#
Let’s use EOReader with products stored in a S3 compatible storage cloud.
This tutorial is a bit tricky as EOReader uses rasterio and cloudpathlib libraries and they are currently handling S3 buckets differently.
That’s why we’ll use the helper context manager temp_s3 from sertit.s3, that will do the right connections for us.
Warning: This tutorial assumes that you have correctly set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY as environment variables
# Imports
import os
from cloudpathlib import AnyPath
import matplotlib.pyplot as plt
from sertit import s3
from eoreader.reader import Reader
from eoreader.bands import MNDWI, CLOUDS
# This endpoint points to your S3 compatible bucket
custom_s3_compatible_endpoint = os.getenv("AWS_S3_ENDPOINT")
# Be sure to have AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables correctly pointing to the credentials of your S3-compatible storage
/home/docs/checkouts/readthedocs.org/user_builds/eoreader/envs/stable/lib/python3.9/site-packages/dask/dataframe/__init__.py:42: FutureWarning:
Dask dataframe query planning is disabled because dask-expr is not installed.
You can install it with `pip install dask[dataframe]` or `conda install dask`.
This will raise in a future version.
warnings.warn(msg, FutureWarning)
with s3.temp_s3(default_endpoint=custom_s3_compatible_endpoint):
path = AnyPath("s3://sertit-eoreader-ci").joinpath(
"optical/S2B_MSIL2A_20200114T065229_N0213_R020_T40REQ_20200114T094749.SAFE/")
# Create the reader
reader = Reader()
# Open your product
prod = reader.open(path, remove_tmp=True)
# Load this band
band_ds = prod.load([MNDWI, CLOUDS])
---------------------------------------------------------------------------
NoCredentialsError Traceback (most recent call last)
Cell In[2], line 9
6 reader = Reader()
8 # Open your product
----> 9 prod = reader.open(path, remove_tmp=True)
11 # Load this band
12 band_ds = prod.load([MNDWI, CLOUDS])
File ~/checkouts/readthedocs.org/user_builds/eoreader/checkouts/stable/eoreader/reader.py:609, in Reader.open(self, product_path, archive_path, output_path, method, remove_tmp, custom, constellation, **kwargs)
606 prod = self._open_stac_item(product_path, output_path, remove_tmp, **kwargs)
607 else:
608 # If not an Item, it should be a path to somewhere
--> 609 prod = self._open_path(
610 product_path,
611 archive_path,
612 output_path,
613 method,
614 remove_tmp,
615 custom,
616 constellation,
617 **kwargs,
618 )
620 if not prod:
621 LOGGER.warning(
622 f"There is no existing products in EOReader corresponding to {product_path}."
623 )
File ~/checkouts/readthedocs.org/user_builds/eoreader/checkouts/stable/eoreader/reader.py:730, in Reader._open_path(self, product_path, archive_path, output_path, method, remove_tmp, custom, constellation, **kwargs)
712 """
713 Open a path or a cloud URI as an EOReader's product
714
(...)
725 Product: EOReader's product
726 """
728 product_path = AnyPath(product_path)
--> 730 if not product_path.exists():
731 FileNotFoundError(f"Non existing product: {product_path}")
733 if custom:
File ~/checkouts/readthedocs.org/user_builds/eoreader/envs/stable/lib/python3.9/site-packages/cloudpathlib/cloudpath.py:431, in CloudPath.exists(self)
430 def exists(self) -> bool:
--> 431 return self.client._exists(self)
File ~/checkouts/readthedocs.org/user_builds/eoreader/envs/stable/lib/python3.9/site-packages/cloudpathlib/s3/s3client.py:181, in S3Client._exists(self, cloud_path)
178 except ClientError:
179 return False
--> 181 return self._s3_file_query(cloud_path) is not None
File ~/checkouts/readthedocs.org/user_builds/eoreader/envs/stable/lib/python3.9/site-packages/cloudpathlib/s3/s3client.py:188, in S3Client._s3_file_query(self, cloud_path)
185 # check if this is an object that we can access directly
186 try:
187 # head_object accepts all download extra args (note: Object.load does not accept extra args so we do not use it for this check)
--> 188 self.client.head_object(
189 Bucket=cloud_path.bucket,
190 Key=cloud_path.key.rstrip("/"),
191 **self.boto3_dl_extra_args,
192 )
193 return "file"
195 # else, confirm it is a dir by filtering to the first item under the prefix plus a "/"
File ~/checkouts/readthedocs.org/user_builds/eoreader/envs/stable/lib/python3.9/site-packages/botocore/client.py:569, in ClientCreator._create_api_method.<locals>._api_call(self, *args, **kwargs)
565 raise TypeError(
566 f"{py_operation_name}() only accepts keyword arguments."
567 )
568 # The "self" in this scope is referring to the BaseClient.
--> 569 return self._make_api_call(operation_name, kwargs)
File ~/checkouts/readthedocs.org/user_builds/eoreader/envs/stable/lib/python3.9/site-packages/botocore/client.py:1005, in BaseClient._make_api_call(self, operation_name, api_params)
1001 maybe_compress_request(
1002 self.meta.config, request_dict, operation_model
1003 )
1004 apply_request_checksum(request_dict)
-> 1005 http, parsed_response = self._make_request(
1006 operation_model, request_dict, request_context
1007 )
1009 self.meta.events.emit(
1010 f'after-call.{service_id}.{operation_name}',
1011 http_response=http,
(...)
1014 context=request_context,
1015 )
1017 if http.status_code >= 300:
File ~/checkouts/readthedocs.org/user_builds/eoreader/envs/stable/lib/python3.9/site-packages/botocore/client.py:1029, in BaseClient._make_request(self, operation_model, request_dict, request_context)
1027 def _make_request(self, operation_model, request_dict, request_context):
1028 try:
-> 1029 return self._endpoint.make_request(operation_model, request_dict)
1030 except Exception as e:
1031 self.meta.events.emit(
1032 f'after-call-error.{self._service_model.service_id.hyphenize()}.{operation_model.name}',
1033 exception=e,
1034 context=request_context,
1035 )
File ~/checkouts/readthedocs.org/user_builds/eoreader/envs/stable/lib/python3.9/site-packages/botocore/endpoint.py:119, in Endpoint.make_request(self, operation_model, request_dict)
113 def make_request(self, operation_model, request_dict):
114 logger.debug(
115 "Making request for %s with params: %s",
116 operation_model,
117 request_dict,
118 )
--> 119 return self._send_request(request_dict, operation_model)
File ~/checkouts/readthedocs.org/user_builds/eoreader/envs/stable/lib/python3.9/site-packages/botocore/endpoint.py:196, in Endpoint._send_request(self, request_dict, operation_model)
194 context = request_dict['context']
195 self._update_retries_context(context, attempts)
--> 196 request = self.create_request(request_dict, operation_model)
197 success_response, exception = self._get_response(
198 request, operation_model, context
199 )
200 while self._needs_retry(
201 attempts,
202 operation_model,
(...)
205 exception,
206 ):
File ~/checkouts/readthedocs.org/user_builds/eoreader/envs/stable/lib/python3.9/site-packages/botocore/endpoint.py:132, in Endpoint.create_request(self, params, operation_model)
130 service_id = operation_model.service_model.service_id.hyphenize()
131 event_name = f'request-created.{service_id}.{operation_model.name}'
--> 132 self._event_emitter.emit(
133 event_name,
134 request=request,
135 operation_name=operation_model.name,
136 )
137 prepared_request = self.prepare_request(request)
138 return prepared_request
File ~/checkouts/readthedocs.org/user_builds/eoreader/envs/stable/lib/python3.9/site-packages/botocore/hooks.py:412, in EventAliaser.emit(self, event_name, **kwargs)
410 def emit(self, event_name, **kwargs):
411 aliased_event_name = self._alias_event_name(event_name)
--> 412 return self._emitter.emit(aliased_event_name, **kwargs)
File ~/checkouts/readthedocs.org/user_builds/eoreader/envs/stable/lib/python3.9/site-packages/botocore/hooks.py:256, in HierarchicalEmitter.emit(self, event_name, **kwargs)
245 def emit(self, event_name, **kwargs):
246 """
247 Emit an event by name with arguments passed as keyword args.
248
(...)
254 handlers.
255 """
--> 256 return self._emit(event_name, kwargs)
File ~/checkouts/readthedocs.org/user_builds/eoreader/envs/stable/lib/python3.9/site-packages/botocore/hooks.py:239, in HierarchicalEmitter._emit(self, event_name, kwargs, stop_on_response)
237 for handler in handlers_to_call:
238 logger.debug('Event %s: calling handler %s', event_name, handler)
--> 239 response = handler(**kwargs)
240 responses.append((handler, response))
241 if stop_on_response and response is not None:
File ~/checkouts/readthedocs.org/user_builds/eoreader/envs/stable/lib/python3.9/site-packages/botocore/signers.py:106, in RequestSigner.handler(self, operation_name, request, **kwargs)
101 def handler(self, operation_name=None, request=None, **kwargs):
102 # This is typically hooked up to the "request-created" event
103 # from a client's event emitter. When a new request is created
104 # this method is invoked to sign the request.
105 # Don't call this method directly.
--> 106 return self.sign(operation_name, request)
File ~/checkouts/readthedocs.org/user_builds/eoreader/envs/stable/lib/python3.9/site-packages/botocore/signers.py:198, in RequestSigner.sign(self, operation_name, request, region_name, signing_type, expires_in, signing_name)
195 else:
196 raise e
--> 198 auth.add_auth(request)
File ~/checkouts/readthedocs.org/user_builds/eoreader/envs/stable/lib/python3.9/site-packages/botocore/auth.py:424, in SigV4Auth.add_auth(self, request)
422 def add_auth(self, request):
423 if self.credentials is None:
--> 424 raise NoCredentialsError()
425 datetime_now = datetime.datetime.utcnow()
426 request.context['timestamp'] = datetime_now.strftime(SIGV4_TIMESTAMP)
NoCredentialsError: Unable to locate credentials
# Display
nrows = len(band_ds)
fig, axes = plt.subplots(nrows=nrows, figsize=(3 * nrows, 6 * nrows), subplot_kw={"box_aspect": 1}) # Square plots
for i, band in enumerate(band_ds.values()):
band[::10, ::10].plot(x="x", y="y", ax=axes[i])
