diff --git a/masterscraper/__init__.py b/masterscraper/__init__.py index 9dfc30e..6237c5a 100644 --- a/masterscraper/__init__.py +++ b/masterscraper/__init__.py @@ -103,13 +103,13 @@ class scrape: self.meta['name'] = self.meta['name'].replace('Gnp', 'GNP') - # Get Key Names + # Get Key Names Search Spaces self.data_info.append( [self.meta['name']+' '+key for key in self.data[0]] ) - # Process Key Names + # Process Variable Key Names key_name = [] - for key in self.data_info[0]: + for key in self.data[0]: if(key.lower().find('country') >=0 or key.lower().find('countries') >=0 or key.lower().find('dependency') >=0 ): @@ -143,8 +143,6 @@ class scrape: tmp_key = re.sub('per\ 100k\ live\ births', '', tmp_key) tmp_key = re.sub('per\ 100k\ population', '', tmp_key) - #print( '"{0}"'.format(tmp_key) ) - tmp_key = tmp_key.strip() tmp_key = tmp_key.replace(' ','.') @@ -158,7 +156,7 @@ class scrape: self.data_info.append( key_name ) - # Process Unit Type + # Process Variable Unit Type key_unit = [] for key in self.data_info[0]: @@ -291,7 +289,7 @@ class scrape: self.data_info.append( key_multiplyer ) - # Get Year + # Get Variable Year key_year = [] for key in self.data_info[0]: if re.match('\d\d\d\d', key): @@ -306,7 +304,7 @@ class scrape: self.data_info.append( key_year ) - # Get Type + # Get Variable Type for key in self.data_info[1]: if key == 'country.name': self.meta['type'] = 'global' elif key == 'year': self.meta['type'] = 'historical' @@ -315,7 +313,7 @@ class scrape: if self.meta['type'] == None: self.meta['type'] = 'unkown' - # Get Scope + # Get Variable Scope key_scope = [] for key in self.data_info[1]: if key.find('male') >=0: key_scope.append( 'male' ) @@ -330,7 +328,7 @@ class scrape: self.data_info.append( key_scope ) - # Get Category + # Get Variable Category search = self.meta['name'].join(self.data_info[0]).lower().strip() diff --git a/scrape_single.py b/scrape_single.py index 4125dca..a8ec956 100644 --- a/scrape_single.py +++ b/scrape_single.py @@ -5,7 +5,7 @@ import masterscraper as ms -scrape = ms.scrape('https://www.macrotrends.net//countries/CHN/china/suicide-rate') +scrape = ms.scrape('https://www.macrotrends.net/countries/AUT/austria/population-density') scrape.get_meta() scrape.clean() scrape.save()