Fix hierarchy_radio error when scraping

Author: jasonbosco

.gitignore

@@ -20,3 +20,4 @@ geckodriver.log
 
 configs/private
 /typesense-server-data/
+typesense-data/

README.md

@@ -37,6 +37,13 @@ Remember to change the version numbers in the URL as needed.
 
 This section only applies if you're making changes to this scraper itself. If you only need to run the scraper, see Usage instructions above.
 
+#### Running the code locally
+
+```shellsession
+$ pipenv shell
+$ ./docsearch run configs/public/typesense_docs.json
+```
+
 #### Releasing a new version
 
 Basic/abbreviated instructions:

configs/public/typesense_docs.json

@@ -5,7 +5,7 @@
       "url": "https://typesense.org/docs/(?P<version>.*?)/",
       "variables": {
         "version": [
-          "0.21.0"
+          "27.1"
         ]
       }
     }
@@ -22,9 +22,13 @@
   },
   "scrape_start_urls": false,
   "strip_chars": " .,;:#",
-  "nb_hits": 505,
   "custom_settings": {
-    "token_separators": ["_"],
-    "symbols_to_index": ["*"]
-  }
+    "token_separators": [
+      "_"
+    ],
+    "symbols_to_index": [
+      "*"
+    ]
+  },
+  "nb_hits": 16502
 }

scraper/src/typesense_helper.py

@@ -209,10 +209,12 @@ def transform_record(record):
 
         # Flatten nested hierarchy fields
         for x in range(0, 7):
-            if record['hierarchy'][f'lvl{x}'] is not None:
-                transformed_record[f'hierarchy.lvl{x}'] = record['hierarchy'][f'lvl{x}']
-            if record['hierarchy_radio'][f'lvl{x}'] is not None:
-                transformed_record[f'hierarchy_radio.lvl{x}'] = record['hierarchy_radio'][f'lvl{x}']
+            if 'hierarchy' in record and f'lvl{x}' in record['hierarchy']:
+                if record['hierarchy'][f'lvl{x}'] is not None:
+                    transformed_record[f'hierarchy.lvl{x}'] = record['hierarchy'][f'lvl{x}']
+            if 'hierarchy_radio' in record and f'lvl{x}' in record['hierarchy_radio']:
+                if record['hierarchy_radio'][f'lvl{x}'] is not None:
+                    transformed_record[f'hierarchy_radio.lvl{x}'] = record['hierarchy_radio'][f'lvl{x}']
 
         # Convert version to array
         if 'version' in record and type(record['version']) == str: