"""ImageNet-A: natural adversarial examples for 200 ImageNet classes."""
import io
import tarfile
from PIL import Image as PILImage
from stable_datasets.images._imagenet_wnids import WNID_TO_IDX
from stable_datasets.schema import ClassLabel, DatasetInfo, DatasetSource, DownloadInfo, Features, Version
from stable_datasets.schema import Image as ImageFeature
from stable_datasets.utils import BaseDatasetBuilder
# The 200 ImageNet classes used by ImageNet-A as (WordNet ID, readable name) pairs, copied
# from the README.txt shipped inside the official archive. They are sorted by WordNet ID,
# which is the same order as the canonical ImageNet-1K list, so label i means entry i here.
_IMAGENET_A_CLASSES: list[tuple[str, str]] = [
("n01498041", "stingray"),
("n01531178", "goldfinch"),
("n01534433", "junco"),
("n01558993", "American robin"),
("n01580077", "jay"),
("n01614925", "bald eagle"),
("n01616318", "vulture"),
("n01631663", "newt"),
("n01641577", "American bullfrog"),
("n01669191", "box turtle"),
("n01677366", "green iguana"),
("n01687978", "agama"),
("n01694178", "chameleon"),
("n01698640", "American alligator"),
("n01735189", "garter snake"),
("n01770081", "harvestman"),
("n01770393", "scorpion"),
("n01774750", "tarantula"),
("n01784675", "centipede"),
("n01819313", "sulphur-crested cockatoo"),
("n01820546", "lorikeet"),
("n01833805", "hummingbird"),
("n01843383", "toucan"),
("n01847000", "duck"),
("n01855672", "goose"),
("n01882714", "koala"),
("n01910747", "jellyfish"),
("n01914609", "sea anemone"),
("n01924916", "flatworm"),
("n01944390", "snail"),
("n01985128", "crayfish"),
("n01986214", "hermit crab"),
("n02007558", "flamingo"),
("n02009912", "great egret"),
("n02037110", "oystercatcher"),
("n02051845", "pelican"),
("n02077923", "sea lion"),
("n02085620", "Chihuahua"),
("n02099601", "Golden Retriever"),
("n02106550", "Rottweiler"),
("n02106662", "German Shepherd Dog"),
("n02110958", "pug"),
("n02119022", "red fox"),
("n02123394", "Persian cat"),
("n02127052", "lynx"),
("n02129165", "lion"),
("n02133161", "American black bear"),
("n02137549", "mongoose"),
("n02165456", "ladybug"),
("n02174001", "rhinoceros beetle"),
("n02177972", "weevil"),
("n02190166", "fly"),
("n02206856", "bee"),
("n02219486", "ant"),
("n02226429", "grasshopper"),
("n02231487", "stick insect"),
("n02233338", "cockroach"),
("n02236044", "mantis"),
("n02259212", "leafhopper"),
("n02268443", "dragonfly"),
("n02279972", "monarch butterfly"),
("n02280649", "small white"),
("n02281787", "gossamer-winged butterfly"),
("n02317335", "starfish"),
("n02325366", "cottontail rabbit"),
("n02346627", "porcupine"),
("n02356798", "fox squirrel"),
("n02361337", "marmot"),
("n02410509", "bison"),
("n02445715", "skunk"),
("n02454379", "armadillo"),
("n02486410", "baboon"),
("n02492035", "white-headed capuchin"),
("n02504458", "African bush elephant"),
("n02655020", "pufferfish"),
("n02669723", "academic gown"),
("n02672831", "accordion"),
("n02676566", "acoustic guitar"),
("n02690373", "airliner"),
("n02701002", "ambulance"),
("n02730930", "apron"),
("n02777292", "balance beam"),
("n02782093", "balloon"),
("n02787622", "banjo"),
("n02793495", "barn"),
("n02797295", "wheelbarrow"),
("n02802426", "basketball"),
("n02814860", "lighthouse"),
("n02815834", "beaker"),
("n02837789", "bikini"),
("n02879718", "bow"),
("n02883205", "bow tie"),
("n02895154", "breastplate"),
("n02906734", "broom"),
("n02948072", "candle"),
("n02951358", "canoe"),
("n02980441", "castle"),
("n02992211", "cello"),
("n02999410", "chain"),
("n03014705", "chest"),
("n03026506", "Christmas stocking"),
("n03124043", "cowboy boot"),
("n03125729", "cradle"),
("n03187595", "rotary dial telephone"),
("n03196217", "digital clock"),
("n03223299", "doormat"),
("n03250847", "drumstick"),
("n03255030", "dumbbell"),
("n03291819", "envelope"),
("n03325584", "feather boa"),
("n03355925", "flagpole"),
("n03384352", "forklift"),
("n03388043", "fountain"),
("n03417042", "garbage truck"),
("n03443371", "goblet"),
("n03444034", "go-kart"),
("n03445924", "golf cart"),
("n03452741", "grand piano"),
("n03483316", "hair dryer"),
("n03584829", "clothes iron"),
("n03590841", "jack-o'-lantern"),
("n03594945", "jeep"),
("n03617480", "kimono"),
("n03666591", "lighter"),
("n03670208", "limousine"),
("n03717622", "manhole cover"),
("n03720891", "maraca"),
("n03721384", "marimba"),
("n03724870", "mask"),
("n03775071", "mitten"),
("n03788195", "mosque"),
("n03804744", "nail"),
("n03837869", "obelisk"),
("n03840681", "ocarina"),
("n03854065", "organ"),
("n03888257", "parachute"),
("n03891332", "parking meter"),
("n03935335", "piggy bank"),
("n03982430", "billiard table"),
("n04019541", "hockey puck"),
("n04033901", "quill"),
("n04039381", "racket"),
("n04067472", "reel"),
("n04086273", "revolver"),
("n04099969", "rocking chair"),
("n04118538", "rugby ball"),
("n04131690", "salt shaker"),
("n04133789", "sandal"),
("n04141076", "saxophone"),
("n04146614", "school bus"),
("n04147183", "schooner"),
("n04179913", "sewing machine"),
("n04208210", "shovel"),
("n04235860", "sleeping bag"),
("n04252077", "snowmobile"),
("n04252225", "snowplow"),
("n04254120", "soap dispenser"),
("n04270147", "spatula"),
("n04275548", "spider web"),
("n04310018", "steam locomotive"),
("n04317175", "stethoscope"),
("n04344873", "couch"),
("n04347754", "submarine"),
("n04355338", "sundial"),
("n04366367", "suspension bridge"),
("n04376876", "syringe"),
("n04389033", "tank"),
("n04399382", "teddy bear"),
("n04442312", "toaster"),
("n04456115", "torch"),
("n04482393", "tricycle"),
("n04507155", "umbrella"),
("n04509417", "unicycle"),
("n04532670", "viaduct"),
("n04540053", "volleyball"),
("n04554684", "washing machine"),
("n04562935", "water tower"),
("n04591713", "wine bottle"),
("n04606251", "shipwreck"),
("n07583066", "guacamole"),
("n07695742", "pretzel"),
("n07697313", "cheeseburger"),
("n07697537", "hot dog"),
("n07714990", "broccoli"),
("n07718472", "cucumber"),
("n07720875", "bell pepper"),
("n07734744", "mushroom"),
("n07749582", "lemon"),
("n07753592", "banana"),
("n07760859", "custard apple"),
("n07768694", "pomegranate"),
("n07831146", "carbonara"),
("n09229709", "bubble"),
("n09246464", "cliff"),
("n09472597", "volcano"),
("n09835506", "baseball player"),
("n11879895", "rapeseed"),
("n12057211", "yellow lady's slipper"),
("n12144580", "corn"),
("n12267677", "acorn"),
]
IMAGENET_A_WNIDS: list[str] = [wnid for wnid, _ in _IMAGENET_A_CLASSES]
IMAGENET_A_CLASS_NAMES: list[str] = [name for _, name in _IMAGENET_A_CLASSES]
# IMAGENET_A_TO_IN1K[i] is the ImageNet-1K class index of ImageNet-A label i.
# Use it to evaluate a 1000-way ImageNet classifier: keep only these 200 logits.
IMAGENET_A_TO_IN1K: list[int] = [WNID_TO_IDX[wnid] for wnid in IMAGENET_A_WNIDS]
_WNID_TO_LABEL: dict[str, int] = {wnid: i for i, wnid in enumerate(IMAGENET_A_WNIDS)}
[docs]
class ImageNetA(BaseDatasetBuilder):
"""ImageNet-A: 7,500 real-world photos that standard ImageNet classifiers get wrong.
ImageNet-A is an evaluation-only robustness benchmark, so it has a single ``test``
split. Labels are integers in [0, 200): label ``i`` is the class
``IMAGENET_A_CLASS_NAMES[i]`` with WordNet ID ``IMAGENET_A_WNIDS[i]``, and
``IMAGENET_A_TO_IN1K[i]`` is its ImageNet-1K index.
"""
VERSION = Version("1.0.0")
SOURCE = DatasetSource(
homepage="https://github.com/hendrycks/natural-adv-examples",
assets={
"test": DownloadInfo(
url="https://people.eecs.berkeley.edu/~hendrycks/imagenet-a.tar",
checksum="sha256:3bb3632277e6ba6392ea64c02ddbf4dd2266c9caffd6bc09c9656d28f012589e",
),
},
citation="""@article{hendrycks2021nae,
title={Natural Adversarial Examples},
author={Dan Hendrycks and Kevin Zhao and Steven Basart and Jacob Steinhardt and Dawn Song},
journal={CVPR},
year={2021}}""",
license="MIT",
)
def _info(self):
return DatasetInfo(
description="""ImageNet-A contains 7,500 unmodified, real-world photos from 200 ImageNet classes,
selected because standard ImageNet classifiers misclassify them. It is a test-only benchmark
for robustness to naturally occurring hard examples.""",
features=Features(
{
"image": ImageFeature(),
"label": ClassLabel(names=IMAGENET_A_CLASS_NAMES),
}
),
supervised_keys=("image", "label"),
homepage=self.SOURCE["homepage"],
citation=self.SOURCE["citation"],
license=self.SOURCE["license"],
)
def _generate_examples(self, data_path, split):
# Archive layout: imagenet-a/<wnid>/<file>.jpg (plus README.txt and folder entries).
# File names encode a model's wrong predictions, so the label comes only from the folder.
with tarfile.open(data_path, "r:*") as archive:
for member in archive:
if not member.isfile() or not member.name.lower().endswith((".jpg", ".jpeg")):
continue
wnid = member.name.split("/")[-2]
label = _WNID_TO_LABEL[wnid] # KeyError on an unexpected folder: fail loudly
image_bytes = archive.extractfile(member).read()
with PILImage.open(io.BytesIO(image_bytes)) as img:
# Keep the original JPEG bytes when already RGB; convert anything else.
image = image_bytes if img.mode == "RGB" else img.convert("RGB")
yield member.name, {"image": image, "label": label}