404Forbiden

✏️ Editing: result_queue.py

class ResultQueue(object):
    """If the aggregator can't be reached for some reason, we want
    to still keep calculating results and store them locally until we can reach the agg
    again."""

# Variables for ResultQueue
    # QUEUE_MAX_RESULTS = 150000     # We put results in a queue if aggregator can't be reached.
    # QUEUE_BATCH_SIZE = 10000       # How many queue results we send back to the aggregator at once

def __init__(self, queue_max_results=None, queue_batch_size=None):
        self.data = []
        self.queue_max_results = queue_max_results
        if self.queue_max_results is None:
            self.queue_max_results = 150000
        self.queue_batch_size = queue_batch_size
        if self.queue_batch_size is None:
            self.queue_batch_size = 10000

def pop(self):
        # remove the oldest results first
        self.data.pop(0)

# Pop only the most recent result
    def pop_latest(self):
        self.data.pop()

def pop_results(self, chunk=None):
        if chunk is None:
            chunk = self.queue_batch_size
        # Pop and return 'chunk' of the newest items
        if len(self.data) <= chunk:
            result = self.data
            self.data = []

else:
            x = len(self.data) - chunk
            result = self.data[x:]
            self.data = self.data[:x]

return result

def push(self, result):
        if len(self.data) == self.queue_max_results:
            self.pop()
        self.data.append(result)

def update(self, results):
        if len(results) + len(self.data) <= self.queue_max_results:
            self.data.extend(results)
        else:
            # Make room so we stay under QUEUE_MAX_RESULTS
            free_space = self.queue_max_results - len(self.data)
            removal = len(results) - free_space
            self.data = self.data[removal:]
            self.data.extend(results)

def isEmpty(self):
        return len(self.data) == 0

def queueLength(self):
        return len(self.data)

Sid Gifari From Gifari Industries - BD Cyber Security Team