Global Streetscapes¶
This tutorial demostrates how to use ZenSVI to access Global Streetscapes models to classify the contexts of your street view images for the following aspects:
Panoramic status: True | False, indicates whether the image is panoramic or not
Lighting condition: day | night | dawn/dusk, indicates the time of day the image is taken
View direction: front/back | side, indicates whether the image is facing the front/back end or the side of the road
Weather: clear | cloudy | rainy | snowy | foggy, indicates the weather in the image
Platform: driving surface | walking surface | cycling surface | railway | fields | tunnel, indicates the type of road/platform surface the image is taken from
Quality: good | slightly poor | very poor, indicates the overall image quality
Glare: True | False, indicates the presence/absence of glare
Reflection: True | False, indicates the presence/absence of windshield reflection
The original repository for Global Streetscapes is located at https://github.com/ualsg/global-streetscapes.
Acknowledgement
Hou Y, Quintana M, Khomiakov M, Yap W, Ouyang J, Ito K, Wang Z, Zhao T, Biljecki F (2024): Global Streetscapes — A comprehensive dataset of 10 million street-level images across 688 cities for urban science and analytics. ISPRS Journal of Photogrammetry and Remote Sensing 215: 216-238. doi:10.1016/j.isprsjprs.2024.06.023
BibTex:
@article{2024_global_streetscapes,
author = {Hou, Yujun and Quintana, Matias and Khomiakov, Maxim and Yap, Winston and Ouyang, Jiani and Ito, Koichi and Wang, Zeyu and Zhao, Tianhong and Biljecki, Filip},
doi = {10.1016/j.isprsjprs.2024.06.023},
journal = {ISPRS Journal of Photogrammetry and Remote Sensing},
pages = {216-238},
title = {Global Streetscapes -- A comprehensive dataset of 10 million street-level images across 688 cities for urban science and analytics},
volume = {215},
year = {2024}
}
Download sample images¶
from huggingface_hub import HfApi, hf_hub_download
import os
def download_folder(repo_id, repo_type, folder_path, local_dir):
"""
Download an entire folder from a huggingface dataset repository.
repo_id : string
The ID of the repository (e.g., 'username/repo_name').
repo_type : string
Type of the repo, dataset or model.
folder_path : string
The path to the folder within the repository.
local_dir : string
Local folder to download the data. This mimics git behaviour
"""
api = HfApi()
# list all files in the repo, keep the ones within folder_path
all_files = api.list_repo_files(repo_id, repo_type=repo_type)
files_list = [f for f in all_files if f.startswith(folder_path)]
# download each of those files
for file_path in files_list:
hf_hub_download(repo_id=repo_id, repo_type=repo_type,
filename=file_path, local_dir=local_dir)
# Download entire data/ folder
repo_id = "NUS-UAL/zensvi_test_data" # you can replace this for other huggingface repos
repo_type = "dataset" # required by the API when the repo is a dataset
folder_path = "input/visualization/batch_images/batch_1" # replace the folder you want within the repo
local_dir = "./demo_data" # the local folder in your computer where it will be downloaded
if not os.path.exists(local_dir):
os.makedirs(local_dir)
# By default, huggingface download them to the .cache/huggingface folder
download_folder(repo_id, repo_type, folder_path, local_dir)
Classify¶
from zensvi.cv import ClassifierPanorama
from zensvi.cv import ClassifierLighting
from zensvi.cv import ClassifierViewDirection
from zensvi.cv import ClassifierWeather
from zensvi.cv import ClassifierPlatform
from zensvi.cv import ClassifierQuality
from zensvi.cv import ClassifierGlare
from zensvi.cv import ClassifierReflection
import os
image_input = f"{local_dir}/{folder_path}"
labels = ['pano', 'lighting', 'view_dir', 'weather', 'platform', 'quality', 'glare', 'reflection']
# Classify panoramic status
label = labels[0]
print(f'Classifying {label}...')
summary_output = f'{local_dir}/output/{label}'
if not os.path.exists(summary_output):
os.makedirs(summary_output)
classifier = ClassifierPanorama()
classifier.classify(
image_input,
dir_summary_output=summary_output,
batch_size=4,
)
# Classify lighting condition
label = labels[1]
print(f'Classifying {label}...')
summary_output = f'{local_dir}/output/{label}'
if not os.path.exists(summary_output):
os.makedirs(summary_output)
classifier = ClassifierLighting()
classifier.classify(
image_input,
dir_summary_output=summary_output,
batch_size=4,
)
# Classify view direction
label = labels[2]
print(f'Classifying {label}...')
summary_output = f'{local_dir}/output/{label}'
if not os.path.exists(summary_output):
os.makedirs(summary_output)
classifier = ClassifierViewDirection()
classifier.classify(
image_input,
dir_summary_output=summary_output,
batch_size=4,
)
# Classify weather
label = labels[3]
print(f'Classifying {label}...')
summary_output = f'{local_dir}/output/{label}'
if not os.path.exists(summary_output):
os.makedirs(summary_output)
classifier = ClassifierWeather()
classifier.classify(
image_input,
dir_summary_output=summary_output,
batch_size=4,
)
# Classify platform
label = labels[4]
print(f'Classifying {label}...')
summary_output = f'{local_dir}/output/{label}'
if not os.path.exists(summary_output):
os.makedirs(summary_output)
classifier = ClassifierPlatform()
classifier.classify(
image_input,
dir_summary_output=summary_output,
batch_size=4,
)
# Classify quality
label = labels[5]
print(f'Classifying {label}...')
summary_output = f'{local_dir}/output/{label}'
if not os.path.exists(summary_output):
os.makedirs(summary_output)
classifier = ClassifierQuality()
classifier.classify(
image_input,
dir_summary_output=summary_output,
batch_size=4,
)
# Classify glare
label = labels[6]
print(f'Classifying {label}...')
summary_output = f'{local_dir}/output/{label}'
if not os.path.exists(summary_output):
os.makedirs(summary_output)
classifier = ClassifierGlare()
classifier.classify(
image_input,
dir_summary_output=summary_output,
batch_size=4,
)
# Classify reflection
label = labels[7]
print(f'Classifying {label}...')
summary_output = f'{local_dir}/output/{label}'
if not os.path.exists(summary_output):
os.makedirs(summary_output)
classifier = ClassifierReflection()
classifier.classify(
image_input,
dir_summary_output=summary_output,
batch_size=4,
)
print('Done.')
Classifying pano...
Using GPU
Using GPU
/data/yujun/miniconda3/envs/zensvi/lib/python3.10/site-packages/zensvi/cv/classification/panorama.py:102: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
checkpoint = torch.load(checkpoint_path, map_location=lambda storage, loc: storage)
/data/yujun/miniconda3/envs/zensvi/lib/python3.10/site-packages/pytorch_lightning/core/saving.py:195: Found keys that are not in the model state dict but in the checkpoint: ['loss_fn.weight']
Classifying panorama: 100%|██████████| 25/25 [00:01<00:00, 17.40it/s]
Classifying lighting...
Using GPU
Using GPU
/data/yujun/miniconda3/envs/zensvi/lib/python3.10/site-packages/zensvi/cv/classification/lighting.py:98: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
checkpoint = torch.load(checkpoint_path, map_location=lambda storage, loc: storage)
/data/yujun/miniconda3/envs/zensvi/lib/python3.10/site-packages/pytorch_lightning/core/saving.py:195: Found keys that are not in the model state dict but in the checkpoint: ['loss_fn.weight']
Classifying lighting: 100%|██████████| 25/25 [00:01<00:00, 18.65it/s]
Classifying view_dir...
Using GPU
Using GPU
/data/yujun/miniconda3/envs/zensvi/lib/python3.10/site-packages/zensvi/cv/classification/view_direction.py:82: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
checkpoint = torch.load(checkpoint_path, map_location=lambda storage, loc: storage)
/data/yujun/miniconda3/envs/zensvi/lib/python3.10/site-packages/pytorch_lightning/core/saving.py:195: Found keys that are not in the model state dict but in the checkpoint: ['loss_fn.weight']
Classifying view_direction: 100%|██████████| 25/25 [00:01<00:00, 20.35it/s]
Classifying weather...
Using GPU
Using GPU
/data/yujun/miniconda3/envs/zensvi/lib/python3.10/site-packages/zensvi/cv/classification/weather.py:82: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
checkpoint = torch.load(checkpoint_path, map_location=lambda storage, loc: storage)
/data/yujun/miniconda3/envs/zensvi/lib/python3.10/site-packages/pytorch_lightning/core/saving.py:195: Found keys that are not in the model state dict but in the checkpoint: ['loss_fn.weight']
Classifying weather: 100%|██████████| 25/25 [00:01<00:00, 22.33it/s]
Classifying platform...
Using GPU
Using GPU
/data/yujun/miniconda3/envs/zensvi/lib/python3.10/site-packages/zensvi/cv/classification/platform.py:82: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
checkpoint = torch.load(checkpoint_path, map_location=lambda storage, loc: storage)
/data/yujun/miniconda3/envs/zensvi/lib/python3.10/site-packages/pytorch_lightning/core/saving.py:195: Found keys that are not in the model state dict but in the checkpoint: ['loss_fn.weight']
Classifying platform: 100%|██████████| 25/25 [00:01<00:00, 22.27it/s]
Classifying quality...
Using GPU
Using GPU
/data/yujun/miniconda3/envs/zensvi/lib/python3.10/site-packages/zensvi/cv/classification/quality.py:82: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
checkpoint = torch.load(checkpoint_path, map_location=lambda storage, loc: storage)
/data/yujun/miniconda3/envs/zensvi/lib/python3.10/site-packages/pytorch_lightning/core/saving.py:195: Found keys that are not in the model state dict but in the checkpoint: ['loss_fn.weight']
Classifying quality: 100%|██████████| 25/25 [00:01<00:00, 24.33it/s]
Classifying glare...
Using GPU
Using GPU
/data/yujun/miniconda3/envs/zensvi/lib/python3.10/site-packages/zensvi/cv/classification/glare.py:106: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
checkpoint = torch.load(checkpoint_path, map_location=lambda storage, loc: storage)
/data/yujun/miniconda3/envs/zensvi/lib/python3.10/site-packages/pytorch_lightning/core/saving.py:195: Found keys that are not in the model state dict but in the checkpoint: ['loss_fn.weight']
Classifying glare: 100%|██████████| 25/25 [00:01<00:00, 21.06it/s]
Classifying reflection...
Using GPU
Using GPU
/data/yujun/miniconda3/envs/zensvi/lib/python3.10/site-packages/zensvi/cv/classification/reflection.py:84: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
checkpoint = torch.load(checkpoint_path, map_location=lambda storage, loc: storage)
/data/yujun/miniconda3/envs/zensvi/lib/python3.10/site-packages/pytorch_lightning/core/saving.py:195: Found keys that are not in the model state dict but in the checkpoint: ['loss_fn.weight']
Classifying reflection: 100%|██████████| 25/25 [00:01<00:00, 22.42it/s]
Done.
Examine output¶
import os
import math
import pandas as pd
import matplotlib.pyplot as plt
from PIL import Image
# Set the paths
output_folders = [
f'{local_dir}/output/{label}' for label in labels
]
# To collect all image IDs
image_ids = set([filename.split('.')[0] for filename in os.listdir(image_input)])
# Number of rows and columns for the grid
cols = 2
rows = math.ceil(len(image_ids) / cols)
# Create a figure with subplots
fig, axes = plt.subplots(rows, cols, figsize=(10, 100)) # Adjust figsize as needed
axes = axes.flatten() # Flatten axes for easier indexing
# Read the classification results into a list of DataFrames
output_dfs = {}
for folder in output_folders:
results_df = pd.read_csv(os.path.join(folder, 'results.csv'))
results_df['filename_key'] = results_df['filename_key'].astype(str)
label = folder.split('/')[-1]
output_dfs[label] = results_df
# Sort image IDs to ensure consistent ordering
sorted_image_ids = sorted(image_ids)
# Iterate through each unique image ID
for i, img_uuid in enumerate(sorted_image_ids): # Limit to available images
img_path = os.path.join(image_input, f"{img_uuid}.png") # Adjust if necessary
if os.path.exists(img_path): # Check if image exists
img = Image.open(img_path)
# Display the image
axes[i].imshow(img)
axes[i].axis('off') # Hide axes for the image
# Prepare to display classification outcomes
outcome_text = []
for key, df in output_dfs.items():
result = df.loc[df.iloc[:, 0] == img_uuid, df.columns[1]].values
if len(result) > 0:
outcome_text.append(f"{key}: {result[0]}")
else:
outcome_text.append(f"{key}: N/A") # If no result found
# Overlay the classification outcomes at the bottom right corner of the image
axes[i].text(0.95, 0.05, "\n".join(outcome_text), fontsize=7, ha='right', va='bottom',
color='white', bbox=dict(facecolor='black', alpha=0.5), transform=axes[i].transAxes) # Add a semi-transparent background for better readability
else:
print(f"Image for UUID {img_uuid} not found.")
# Adjust layout and remove empty subplots
for j in range(len(sorted_image_ids), rows * cols):
axes[j].axis('off') # Hide any unused axes
plt.tight_layout()
plt.show()
From the above we can see that the models can accurately distinguish panorama/non-panorama, and get most of the lighting condition correct. View direction is mostly correct for perspective images, while for panoramas we can ignore this label because it is not applicable since a panorama is all-directional. The same goes for quality, where the model does consistently well for perspective images but for panoramas, the high level of distortion (due to projection) present in them might have confused the model to deem them as low quality (in reality, panoramas are usually of high quality, so the quality model is more relevant for filtering perspective images). Weather is mostly correctly predicted other than few occasions where the model confuses between cloudy and foggy. Glare and reflection are mostly consistent with observation. For platform, the model tends to mix sidewalks and cycling paths (though one usually can do both walking and cycling on the same type of surface), and confuse wide roads with fields (especially for images taken with ultra-wide angle or panoramic cameras). For photos taken from a car at a usual angle (which is the most common type of street-level images), they are correctly identified as being taken from a driving surface.