You can see the request URL, and then the next page url. The next page url doesn’t have all the information that the request does, like the season and region in this example. So we’ll need to add that in? Is that the expected behavior? Or do we just need to keep track of the pages? I thought we could just use the next url initially.
Thanks!
Here’s my python code for getting data from RobotEvents:
def _get_robotevents_data(the_url, max_attempts=6, min_delay=65, delay_increment=10):
df_retval = None
got_data = False
num_attempts = 0
while num_attempts < max_attempts and got_data == False:
use_header = random.choice(
ALL_ROBOTEVENTS_HEADERS
) # Randomly selects equal probability
try:
response = requests.get(the_url, headers=use_header)
if response.status_code == 429: # Rate limited
current_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
print(
f"\tRate limited at {current_time}, raising exception....",
flush=True,
)
raise Exception(
f"Rate limited at {current_time} on request to {the_url} with header {str(use_header)}"
)
json_results = json.loads(response.content.decode("utf-8", "ignore"))
except Exception as e:
stack_trace = traceback.format_exc()
print(f"Exception in _get_robotevents_data: {str(e)}", flush=True)
num_attempts = num_attempts + 1
if num_attempts < max_attempts:
time.sleep(min_delay + (num_attempts) * delay_increment)
continue
try:
df_retval = pd.read_json(StringIO(json.dumps(json_results["data"])))
got_data = True
except Exception as e:
num_attempts = num_attempts + 1
print(
"Exception " + str(e) + " with response: " + json.dumps(json_results),
flush=True,
)
if num_attempts < max_attempts:
time.sleep((num_attempts) * delay_increment)
if got_data:
while json_results["meta"]["next_page_url"] is not None:
got_data = False
while num_attempts < max_attempts and got_data == False:
use_header = random.choice(
ALL_ROBOTEVENTS_HEADERS
) # Randomly selects equal probability
try:
response = requests.get(
json_results["meta"]["next_page_url"], headers=use_header
)
if response.status_code == 429: # Rate limited
current_time = datetime.datetime.now().strftime(
"%Y-%m-%d %H:%M:%S"
)
print(
f"\tRate limited at {current_time}, raising exception....",
flush=True,
)
raise Exception(
f"Rate limited on request to {json_results['meta']['next_page_url']} with header {str(use_header)}"
)
json_results = json.loads(
response.content.decode("utf-8", "ignore")
)
except:
json_results = dict()
if "data" in json_results.keys():
result_json = json.dumps(json_results["data"])
df_thispage = pd.read_json(StringIO(result_json))
df_retval = pd.concat([df_retval, df_thispage])
got_data = True
else:
current_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
print(
f"At {current_time} no element 'data' in response: "
+ json.dumps(json_results),
flush=True,
)
print("\tHTTP Response: " + str(response), flush=True)
print(
"\tHTTP Response Headers: " + str(response.headers), flush=True
)
print(
"\tHTTP Response Headers: " + str(response.headers), flush=True
)
num_attempts = num_attempts + 1
if num_attempts >= max_attempts:
raise Exception(
"No element 'data' in RobotEvents response after "
+ str(num_attempts)
+ " attempts."
)
else:
print(f"\tHeader: {str(use_header)}", flush=True)
print(
"Waiting "
+ str(min_delay + (num_attempts) * delay_increment)
+ " seconds for next pull request",
flush=True,
)
time.sleep(min_delay + (num_attempts) * delay_increment)
else:
raise Exception("Unable to get data.")
return df_retval