Phase 1 - Data Format

Phase 1 establishes the foundational data structures that enable all subsequent functionality in the WIA Biodiversity Index Standard. By defining precise, machine-readable schemas for biodiversity information, this phase solves the data fragmentation problem and creates the interoperability essential for global-scale conservation efforts. This chapter details the core data models, validation rules, and implementation guidance for Phase 1 compliance.

Core Schema Architecture

The WIA data format consists of five interconnected core schemas (species occurrence, eDNA sample, habitat classification, diversity index result, and taxonomic reference), each capturing a distinct aspect of biodiversity information while maintaining referential integrity across the system.

Species Occurrence Schema

The species occurrence schema represents the fundamental unit of biodiversity data - a record that a particular species was observed at a specific place and time. This schema extends Darwin Core while adding fields that support modern biodiversity monitoring needs.

{
  "$schema": "https://wia.org/schemas/occurrence/v1.0",
  "occurrence_id": "OCC-2025-123456",
  "dataset_id": "DS-RAINFOREST-2025",

  "species": {
    "scientific_name": "Morpho menelaus",
    "common_name": "Blue Morpho Butterfly",
    "taxonomy": {
      "kingdom": "Animalia",
      "phylum": "Arthropoda",
      "class": "Insecta",
      "order": "Lepidoptera",
      "family": "Nymphalidae",
      "genus": "Morpho",
      "species": "menelaus",
      "subspecies": null
    },
    "taxonomic_authority": "Linnaeus, 1758",
    "taxon_id": "GBIF:5133523",
    "iucn_status": "LC",
    "endemic": false
  },

  "location": {
    "latitude": -2.9500,
    "longitude": -59.9500,
    "coordinate_uncertainty_m": 10,
    "datum": "WGS84",
    "elevation_m": 45,
    "depth_m": null,
    "locality": "Reserva Florestal Adolpho Ducke",
    "country": "Brazil",
    "state_province": "Amazonas",
    "protected_area": true,
    "habitat_type": "tropical_rainforest",
    "habitat_detail": "primary_forest_canopy"
  },

  "temporal": {
    "observation_date": "2025-11-15T09:23:45Z",
    "observation_duration_minutes": 180,
    "season": "wet",
    "time_of_day": "morning"
  },

  "observation": {
    "basis_of_record": "human_observation",
    "sampling_protocol": "transect_walk",
    "individual_count": 3,
    "life_stage": "adult",
    "sex": "unknown",
    "behavior": "nectaring",
    "observer_id": "OBS-2025-BZ047",
    "observer_name": "Dr. Maria Santos",
    "identification_confidence": 0.98,
    "identification_method": "visual",
    "identification_verified_by": null,
    "evidence": {
      "has_photo": true,
      "has_sound": false,
      "has_specimen": false,
      "photo_urls": [
        "https://storage.wia.org/obs/2025/11/15/morpho_001.jpg"
      ]
    }
  },

  "environmental": {
    "temperature_c": 28.5,
    "humidity_percent": 82,
    "cloud_cover_percent": 40,
    "wind_speed_ms": 1.2,
    "canopy_cover_percent": 95
  },

  "quality": {
    "quality_flag": "validated",
    "quality_checks": [
      "coordinate_valid",
      "taxonomy_verified",
      "date_reasonable",
      "habitat_match"
    ],
    "validation_date": "2025-11-16T14:30:00Z",
    "validator_id": "VAL-2025-042"
  },

  "provenance": {
    "data_source": "field_survey",
    "project_name": "Amazon Butterfly Monitoring Program",
    "funding_source": "CNPq (Conselho Nacional de Desenvolvimento Científico e Tecnológico)",
    "permit_number": "ICMBio-12345-2025",
    "created_date": "2025-11-15T12:00:00Z",
    "modified_date": "2025-11-16T14:30:00Z",
    "license": "CC-BY-4.0"
  }
}

Required vs. Optional Fields

The schema defines three tiers of fields to balance completeness with practical constraints:

eDNA Sample Schema

Environmental DNA (eDNA) monitoring represents a revolutionary approach to biodiversity assessment. The eDNA schema captures both field collection and laboratory processing information.

{
  "$schema": "https://wia.org/schemas/edna/v1.0",
  "sample_id": "EDNA-2025-UK-089234",
  "collection": {
    "date": "2025-10-22T11:15:00Z",
    "location": {
      "latitude": 51.4545,
      "longitude": -0.9782,
      "water_body_name": "River Thames",
      "water_body_type": "river",
      "site_description": "River Thames at Reading, town-centre reach"
    },
    "sampling_method": "filtration",
    "volume_liters": 2.0,
    "filter_pore_size_um": 0.45,
    "filter_type": "cellulose_nitrate",
    "replicates": 3,
    "water_parameters": {
      "temperature_c": 12.3,
      "ph": 7.8,
      "conductivity_us_cm": 520,
      "turbidity_ntu": 15
    }
  },

  "processing": {
    "extraction_date": "2025-10-23",
    "extraction_method": "qiagen_dneasy_powerwater",
    "extraction_kit_lot": "QIA-2025-08-B14",
    "dna_concentration_ng_ul": 8.5,
    "dna_quality_260_280": 1.85,
    "sequencing_platform": "illumina_novaseq_6000",
    "sequencing_date": "2025-10-28",
    "target_gene": "COI",
    "primer_set": "mlCOIintF/jgHCO2198",
    "pcr_cycles": 35,
    "sequencing_depth": "50000_reads_per_sample"
  },

  "results": {
    "total_reads": 52847,
    "quality_filtered_reads": 48932,
    "unique_asvs": 247,
    "detected_taxa": [
      {
        "scientific_name": "Salmo trutta",
        "common_name": "Brown Trout",
        "read_count": 15432,
        "asv_count": 8,
        "confidence": 0.97,
        "taxonomy_database": "BOLD_v4.5"
      },
      {
        "scientific_name": "Rutilus rutilus",
        "common_name": "Common Roach",
        "read_count": 8921,
        "asv_count": 5,
        "confidence": 0.94,
        "taxonomy_database": "BOLD_v4.5"
      },
      {
        "scientific_name": "Anguilla anguilla",
        "common_name": "European Eel",
        "read_count": 1247,
        "asv_count": 2,
        "confidence": 0.99,
        "taxonomy_database": "BOLD_v4.5"
      }
    ]
  },

  "quality": {
    "negative_control_contamination": false,
    "positive_control_success": true,
    "primer_dimer_percentage": 3.2,
    "quality_flag": "passed"
  }
}

Habitat Classification Schema

Standardized habitat classification is essential for meaningful biodiversity comparisons. The WIA schema aligns with IUCN habitat classification while adding detail useful for specific ecosystems.

Diversity Index Results Schema

Calculated biodiversity indices must be stored with full metadata about calculation methods and input data to ensure reproducibility.

{
  "$schema": "https://wia.org/schemas/diversity-index/v1.0",
  "calculation_id": "CALC-2025-TH-445",
  "dataset_id": "DS-THAMES-2025-Q4",
  "calculation_date": "2026-01-05T10:30:00Z",

  "spatial_extent": {
    "type": "polygon",
    "coordinates": [/* GeoJSON polygon */],
    "area_km2": 125.8
  },

  "temporal_extent": {
    "start_date": "2025-10-01",
    "end_date": "2025-12-31",
    "duration_days": 92
  },

  "input_data": {
    "occurrence_count": 1847,
    "unique_species": 67,
    "total_individuals": 4523,
    "sampling_effort_hours": 240,
    "observer_count": 12
  },

  "indices": {
    "species_richness": {
      "value": 67,
      "rarefied_to_n": 1000,
      "rarefied_value": 58.3,
      "confidence_interval_95": [54.1, 62.8]
    },
    "shannon_diversity": {
      "value": 3.247,
      "confidence_interval_95": [3.102, 3.398],
      "calculation_method": "natural_log",
      "bootstrap_iterations": 1000
    },
    "simpson_index": {
      "value": 0.0521,
      "diversity_1_minus_d": 0.9479,
      "inverse_simpson": 19.19,
      "confidence_interval_95": [0.0487, 0.0559]
    },
    "pielou_evenness": {
      "value": 0.772,
      "interpretation": "moderately_even"
    },
    "margalef_richness": {
      "value": 7.84
    }
  },

  "comparison": {
    "previous_period": "2025-Q3",
    "richness_change_percent": -5.2,
    "shannon_change": -0.154,
    "trend": "declining",
    "statistical_significance": 0.032
  }
}

Taxonomy Reference Schema

Taxonomic information must be traceable to authoritative sources and versioned to handle updates in classification.

{
  "taxon_id": "WIA-TAX-001234",
  "scientific_name": "Panthera tigris tigris",
  "rank": "subspecies",
  "parent_taxon_id": "WIA-TAX-001200",
  "accepted_name": true,
  "synonyms": [
    "Panthera tigris bengalensis"
  ],
  "common_names": [
    {
      "name": "Bengal Tiger",
      "language": "en"
    },
    {
      "name": "बंगाल टाइगर",
      "language": "hi"
    }
  ],
  "conservation_status": {
    "iucn_category": "EN",
    "iucn_version": "2024-1",
    "population_trend": "Increasing",
    "assessment_date": "2023-06-15"
  },
  "external_ids": {
    "gbif": "7059276",
    "ncbi": "74535"
  },
  "version": "2025-11-01",
  "source": "WIA Taxonomic Authority v1.2"
}

Validation Rules and Quality Assurance

Each schema includes built-in validation rules that can be automatically checked:

Coordinate Validation

• Latitude between -90 and 90
• Longitude between -180 and 180
• Coordinates not at (0,0) unless genuinely in Gulf of Guinea
• Coordinates match stated country (using polygon lookup)
• Elevation reasonable for location (DEM cross-check)

Temporal Validation

• Date not in future
• Date not before 1600 (pre-scientific era)
• Temporal resolution matches precision (day/month/year)
• Seasonal designation matches hemisphere and date

Taxonomic Validation

• Scientific name matches nomenclatural rules
• Taxonomy hierarchy consistent (e.g., mammal in Animalia kingdom)
• Taxon ID resolves to valid external reference

Ecological Validation

• Species occurrence within known range (with flag for outliers)
• Habitat type plausible for species
• Environmental parameters within reasonable bounds
• Individual counts not biologically implausible

Data Exchange Formats

While JSON is the canonical format, the standard supports multiple serialization formats for different use cases:

Implementation Guidance

Database Schema Design

For PostgreSQL implementation, the recommended approach uses JSONB columns for flexible nested data with indexed fields for query performance:

CREATE TABLE occurrences (
  occurrence_id TEXT PRIMARY KEY,
  scientific_name TEXT NOT NULL,
  location GEOMETRY(Point, 4326) NOT NULL,
  observation_date TIMESTAMP NOT NULL,
  data JSONB NOT NULL
);

-- Indexes for common queries
CREATE INDEX idx_species ON occurrences(scientific_name);
CREATE INDEX idx_location ON occurrences USING GIST(location);
CREATE INDEX idx_date ON occurrences(observation_date);
CREATE INDEX idx_habitat ON occurrences((data->'location'->>'habitat_type'));

Migration from Legacy Data

The WIA standard provides migration tools for common legacy formats. Example migration from simple CSV:

import pandas as pd
import wia_biodiversity as wia

# Load legacy CSV
legacy_data = pd.read_csv('old_observations.csv')

# Convert to WIA format
wia_data = wia.convert.from_csv(
    legacy_data,
    field_mapping={
        'Species': 'species.scientific_name',
        'Lat': 'location.latitude',
        'Lon': 'location.longitude',
        'Date': 'temporal.observation_date',
        'Count': 'observation.individual_count'
    },
    defaults={
        'observation.basis_of_record': 'human_observation',
        'location.datum': 'WGS84'
    }
)

# Validate
validation_report = wia.validate(wia_data)
print(f"Valid: {validation_report.valid_count}")
print(f"Errors: {validation_report.error_count}")