AWS#
Let’s read satellite data stored on AWS.
This tutorial is a bit tricky as EOReader uses rasterio and cloudpathlib libraries and they are currently handling S3 buckets differently.
That’s why we’ll use the helper context manager temp_s3 from sertit.s3, that will do the right connections for us.
Note: These Sentinel-2 products are not stored in the .SAFE format. Landsat format is however the same.
Cloud data processed by Element84: Sentinel-2 L2A as COGs#
See this registry (arn:aws:s3:::sentinel-cogs).
This registry is open access, so you don’t have to sign-in.
# Imports
import os
import tempenv
import logging
from sertit import logs, s3
from eoreader.reader import Reader, Constellation
from eoreader.bands import BLUE
/home/docs/checkouts/readthedocs.org/user_builds/eoreader/envs/latest/lib/python3.9/site-packages/dask/dataframe/__init__.py:42: FutureWarning:
Dask dataframe query planning is disabled because dask-expr is not installed.
You can install it with `pip install dask[dataframe]` or `conda install dask`.
This will raise in a future version.
warnings.warn(msg, FutureWarning)
with tempenv.TemporaryEnvironment({
"AWS_S3_ENDPOINT": "s3.us-west-2.amazonaws.com",
}):
with s3.temp_s3(no_sign_request=True):
logs.init_logger(logging.getLogger("eoreader"), logging.DEBUG)
path = r"s3://sentinel-cogs/sentinel-s2-l2a-cogs/39/K/ZU/2023/10/S2A_39KZU_20231031_0_L2A"
prod = Reader().open(path)
prod.plot()
blue = prod.load(BLUE)[BLUE]
2025-05-05 11:13:47,443 - [DEBUG] - Loading bands ['BLUE']
2025-05-05 11:13:47,508 - [DEBUG] - Read BLUE
2025-05-05 11:13:48,290 - [DEBUG] - Manage nodata for band BLUE
2025-05-05 11:13:48,291 - [DEBUG] - Converting BLUE to reflectance
blue[:, ::10, ::10].plot(cmap="Blues_r")
<matplotlib.collections.QuadMesh at 0x7abe64319d60>
Cloud data processed by Sinergise: Sentinel-2 L1C#
See this registry (arn:aws:s3:::sentinel-s2-l1c).
This registry needs authentication.
NB: L2A would have been the same (arn:aws:s3:::sentinel-s2-l2a)
Note: Sinergise data are stored as requester pays in AWS. Don’t forget to state this when requesting data!
with tempenv.TemporaryEnvironment({
"AWS_S3_ENDPOINT": "s3.eu-central-1.amazonaws.com",
"AWS_SECRET_ACCESS_KEY": os.getenv("AWS_S3_AWS_SECRET_ACCESS_KEY"),
"AWS_ACCESS_KEY_ID": os.getenv("AWS_S3_AWS_ACCESS_KEY_ID"),
}):
with s3.temp_s3(requester_pays=True):
path = r"s3://sentinel-s2-l1c/tiles/10/S/DG/2022/7/8/0"
prod = Reader().open(path, constellation=Constellation.S2_SIN)
prod.plot()
---------------------------------------------------------------------------
NoCredentialsError Traceback (most recent call last)
Cell In[4], line 8
6 with s3.temp_s3(requester_pays=True):
7 path = r"s3://sentinel-s2-l1c/tiles/10/S/DG/2022/7/8/0"
----> 8 prod = Reader().open(path, constellation=Constellation.S2_SIN)
9 prod.plot()
File ~/checkouts/readthedocs.org/user_builds/eoreader/checkouts/latest/eoreader/reader.py:609, in Reader.open(self, product_path, archive_path, output_path, method, remove_tmp, custom, constellation, **kwargs)
606 prod = self._open_stac_item(product_path, output_path, remove_tmp, **kwargs)
607 else:
608 # If not an Item, it should be a path to somewhere
--> 609 prod = self._open_path(
610 product_path,
611 archive_path,
612 output_path,
613 method,
614 remove_tmp,
615 custom,
616 constellation,
617 **kwargs,
618 )
620 if not prod:
621 LOGGER.warning(
622 f"There is no existing products in EOReader corresponding to {product_path}."
623 )
File ~/checkouts/readthedocs.org/user_builds/eoreader/checkouts/latest/eoreader/reader.py:730, in Reader._open_path(self, product_path, archive_path, output_path, method, remove_tmp, custom, constellation, **kwargs)
712 """
713 Open a path or a cloud URI as an EOReader's product
714
(...)
725 Product: EOReader's product
726 """
728 product_path = AnyPath(product_path)
--> 730 if not product_path.exists():
731 FileNotFoundError(f"Non existing product: {product_path}")
733 if custom:
File ~/checkouts/readthedocs.org/user_builds/eoreader/envs/latest/lib/python3.9/site-packages/cloudpathlib/cloudpath.py:431, in CloudPath.exists(self)
430 def exists(self) -> bool:
--> 431 return self.client._exists(self)
File ~/checkouts/readthedocs.org/user_builds/eoreader/envs/latest/lib/python3.9/site-packages/cloudpathlib/s3/s3client.py:181, in S3Client._exists(self, cloud_path)
178 except ClientError:
179 return False
--> 181 return self._s3_file_query(cloud_path) is not None
File ~/checkouts/readthedocs.org/user_builds/eoreader/envs/latest/lib/python3.9/site-packages/cloudpathlib/s3/s3client.py:188, in S3Client._s3_file_query(self, cloud_path)
185 # check if this is an object that we can access directly
186 try:
187 # head_object accepts all download extra args (note: Object.load does not accept extra args so we do not use it for this check)
--> 188 self.client.head_object(
189 Bucket=cloud_path.bucket,
190 Key=cloud_path.key.rstrip("/"),
191 **self.boto3_dl_extra_args,
192 )
193 return "file"
195 # else, confirm it is a dir by filtering to the first item under the prefix plus a "/"
File ~/checkouts/readthedocs.org/user_builds/eoreader/envs/latest/lib/python3.9/site-packages/botocore/client.py:569, in ClientCreator._create_api_method.<locals>._api_call(self, *args, **kwargs)
565 raise TypeError(
566 f"{py_operation_name}() only accepts keyword arguments."
567 )
568 # The "self" in this scope is referring to the BaseClient.
--> 569 return self._make_api_call(operation_name, kwargs)
File ~/checkouts/readthedocs.org/user_builds/eoreader/envs/latest/lib/python3.9/site-packages/botocore/client.py:1005, in BaseClient._make_api_call(self, operation_name, api_params)
1001 maybe_compress_request(
1002 self.meta.config, request_dict, operation_model
1003 )
1004 apply_request_checksum(request_dict)
-> 1005 http, parsed_response = self._make_request(
1006 operation_model, request_dict, request_context
1007 )
1009 self.meta.events.emit(
1010 f'after-call.{service_id}.{operation_name}',
1011 http_response=http,
(...)
1014 context=request_context,
1015 )
1017 if http.status_code >= 300:
File ~/checkouts/readthedocs.org/user_builds/eoreader/envs/latest/lib/python3.9/site-packages/botocore/client.py:1029, in BaseClient._make_request(self, operation_model, request_dict, request_context)
1027 def _make_request(self, operation_model, request_dict, request_context):
1028 try:
-> 1029 return self._endpoint.make_request(operation_model, request_dict)
1030 except Exception as e:
1031 self.meta.events.emit(
1032 f'after-call-error.{self._service_model.service_id.hyphenize()}.{operation_model.name}',
1033 exception=e,
1034 context=request_context,
1035 )
File ~/checkouts/readthedocs.org/user_builds/eoreader/envs/latest/lib/python3.9/site-packages/botocore/endpoint.py:119, in Endpoint.make_request(self, operation_model, request_dict)
113 def make_request(self, operation_model, request_dict):
114 logger.debug(
115 "Making request for %s with params: %s",
116 operation_model,
117 request_dict,
118 )
--> 119 return self._send_request(request_dict, operation_model)
File ~/checkouts/readthedocs.org/user_builds/eoreader/envs/latest/lib/python3.9/site-packages/botocore/endpoint.py:196, in Endpoint._send_request(self, request_dict, operation_model)
194 context = request_dict['context']
195 self._update_retries_context(context, attempts)
--> 196 request = self.create_request(request_dict, operation_model)
197 success_response, exception = self._get_response(
198 request, operation_model, context
199 )
200 while self._needs_retry(
201 attempts,
202 operation_model,
(...)
205 exception,
206 ):
File ~/checkouts/readthedocs.org/user_builds/eoreader/envs/latest/lib/python3.9/site-packages/botocore/endpoint.py:132, in Endpoint.create_request(self, params, operation_model)
130 service_id = operation_model.service_model.service_id.hyphenize()
131 event_name = f'request-created.{service_id}.{operation_model.name}'
--> 132 self._event_emitter.emit(
133 event_name,
134 request=request,
135 operation_name=operation_model.name,
136 )
137 prepared_request = self.prepare_request(request)
138 return prepared_request
File ~/checkouts/readthedocs.org/user_builds/eoreader/envs/latest/lib/python3.9/site-packages/botocore/hooks.py:412, in EventAliaser.emit(self, event_name, **kwargs)
410 def emit(self, event_name, **kwargs):
411 aliased_event_name = self._alias_event_name(event_name)
--> 412 return self._emitter.emit(aliased_event_name, **kwargs)
File ~/checkouts/readthedocs.org/user_builds/eoreader/envs/latest/lib/python3.9/site-packages/botocore/hooks.py:256, in HierarchicalEmitter.emit(self, event_name, **kwargs)
245 def emit(self, event_name, **kwargs):
246 """
247 Emit an event by name with arguments passed as keyword args.
248
(...)
254 handlers.
255 """
--> 256 return self._emit(event_name, kwargs)
File ~/checkouts/readthedocs.org/user_builds/eoreader/envs/latest/lib/python3.9/site-packages/botocore/hooks.py:239, in HierarchicalEmitter._emit(self, event_name, kwargs, stop_on_response)
237 for handler in handlers_to_call:
238 logger.debug('Event %s: calling handler %s', event_name, handler)
--> 239 response = handler(**kwargs)
240 responses.append((handler, response))
241 if stop_on_response and response is not None:
File ~/checkouts/readthedocs.org/user_builds/eoreader/envs/latest/lib/python3.9/site-packages/botocore/signers.py:106, in RequestSigner.handler(self, operation_name, request, **kwargs)
101 def handler(self, operation_name=None, request=None, **kwargs):
102 # This is typically hooked up to the "request-created" event
103 # from a client's event emitter. When a new request is created
104 # this method is invoked to sign the request.
105 # Don't call this method directly.
--> 106 return self.sign(operation_name, request)
File ~/checkouts/readthedocs.org/user_builds/eoreader/envs/latest/lib/python3.9/site-packages/botocore/signers.py:198, in RequestSigner.sign(self, operation_name, request, region_name, signing_type, expires_in, signing_name)
195 else:
196 raise e
--> 198 auth.add_auth(request)
File ~/checkouts/readthedocs.org/user_builds/eoreader/envs/latest/lib/python3.9/site-packages/botocore/auth.py:424, in SigV4Auth.add_auth(self, request)
422 def add_auth(self, request):
423 if self.credentials is None:
--> 424 raise NoCredentialsError()
425 datetime_now = datetime.datetime.utcnow()
426 request.context['timestamp'] = datetime_now.strftime(SIGV4_TIMESTAMP)
NoCredentialsError: Unable to locate credentials
Cloud data processed by USGS: Landsat-8#
See this registry (arn:aws:s3:::usgs-landsat).
This registry needs authentication.
Note: USGS data are stored as requester pays in AWS. Don’t forget to state this when requesting data!
with tempenv.TemporaryEnvironment({
"AWS_S3_ENDPOINT": "s3.us-west-2.amazonaws.com",
"AWS_SECRET_ACCESS_KEY": os.getenv("AWS_S3_AWS_SECRET_ACCESS_KEY"),
"AWS_ACCESS_KEY_ID": os.getenv("AWS_S3_AWS_ACCESS_KEY_ID"),
}):
with s3.temp_s3(requester_pays=True):
path = r"s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1"
prod = Reader().open(path)
prod.plot()