Skip to content

Help us improve this website by doing a short survey.

How to export all datasets from data.govt.nz into a CSV

Using the ckanapi-exporter tool you can extract all dataset metadata into a single CSV file.

Requirements

Steps

  1. Install requirements
  2. Create a columns.json file (sets out some preset data propoerties to extract from data.govt.nz API, you can customise if required). See below for file contents.
  3. Run the below command on your terminal
ckanapi-exporter --url 'https://catalogue.data.govt.nz' --columns columns.json > datasets.csv

columns.json

{
 "Title": {
     "pattern": "^title$"
 },
 "Agency": {
     "pattern": ["^organization$", "^title$"]
 },
 "URL": {
     "pattern": "^url$"
 },
 "CatalogueCreated": {
     "pattern": "^metadata_created$",
     "max_length": 10
 },
 "CatalogueLastUpdated": {
     "pattern": "^metadata_modified$",
     "max_length": 10
 },
 "DatasetCreated": {
     "pattern": "^issued$",
     "max_length": 10
 },
 "DatasetLastUpdated": {
     "pattern": "^modified$",
     "max_length": 10
 },
 "FrequencyOfUpdate": {
     "pattern": "^frequency_of_update$"
 },
 "Rights": {
     "pattern": "^license_title$"
 },
 "FormatsAvailable": {
     "pattern": ["^resources$", "^format$"],
     "case_sensitive": true,
     "deduplicate": true
 },
 "Description": {
     "pattern": "^notes$"
 },
 "Tags": {
   "pattern": ["^tags$", "^display_name$"]
 },
 "Groups": {
     "pattern": ["^groups$", "^display_name$"]
 },
 "AgencyContact": {
     "pattern": "^author$"
 },
 "AgencyContactEmail":{
     "pattern": "^author_email$"
 },
 "AgencyContactPhone":{
     "pattern": "^author_phone$"
 },
 "DatasetContact": {
     "pattern": "^maintainer$"
 },
 "DatasetContactEmail": {
     "pattern": "^maintainer_email$"
 },
 "DatasetContactPhone": {
     "pattern": "^maintainer_phone$"
 },
 "PermanentIdentifier":{
     "pattern": "^id$"
 },
 "SourceIdentifier": {
     "pattern": "^source_identifier$"
 }
}

Top