Index

A

AbuseType (class in helpers.mongohelper) ABUSIVE_BOOLEAN (helpers.mongohelper.AbuseType attribute) ABUSIVE_BOOLEAN_COUNT (helpers.mongohelper.Properties attribute) ABUSIVE_NUMBER (helpers.mongohelper.AbuseType attribute) ABUSIVE_NUMBER_COUNT (helpers.mongohelper.Properties attribute) act_end_array() (in module helpers.nodeextractor) act_end_map() (in module helpers.nodeextractor) act_leaf() (in module helpers.nodeextractor) act_map_key() (in module helpers.nodeextractor) act_start_array() (in module helpers.nodeextractor) act_start_map() (in module helpers.nodeextractor) add_file() (helpers.classes.node.Node method) add_node() (helpers.mongohelper.Client method) (in module webapp.routes) add_object() (helpers.classes.arrayproperties.ArrayProperties method) add_repository() (helpers.mongohelper.Client method) (in module scraper) add_repository_route() (in module webapp.routes)

add_repository_stats() (helpers.mongohelper.Client method) add_scraped_urls() (helpers.mongohelper.Client method) add_simple() (helpers.classes.arrayproperties.ArrayProperties method) ADDED (helpers.classes.repositorystatus.RepositoryStatus attribute) analysis module ANALYSIS (helpers.classes.job.JobType attribute) analysis() (helpers.classes.arrayproperties.ArrayProperties method) ANALYSIS_DURATION (helpers.mongohelper.DurationType attribute) AnalysisErrors (class in helpers.classes.errors) AnalysisJob (class in helpers.classes.job) analyze_repository() (in module webapp.routes) ANALYZED (helpers.classes.repositorystatus.RepositoryStatus attribute) ANALYZING (helpers.classes.repositorystatus.RepositoryStatus attribute) ARRAY (helpers.classes.nodetype.NodeType attribute) ARRAY_COUNT (helpers.mongohelper.Properties attribute) ARRAY_ITEM (helpers.classes.nodetype.NodeType attribute) ArrayProperties (class in helpers.classes.arrayproperties)

B

BLACK (helpers.colors.TColorsBackground attribute) (helpers.colors.TColorsForeground attribute) BLUE (helpers.colors.TColorsBackground attribute) (helpers.colors.TColorsForeground attribute)

BOLD (helpers.colors.TStyles attribute) BOOLEAN (helpers.classes.nodetype.NodeType attribute) BOOLEAN_COUNT (helpers.mongohelper.Properties attribute)

C

check_for_nodes() (in module main_server) CKAN (helpers.classes.errors.ScrapingErrors attribute) Client (class in helpers.mongohelper) COLLECTION_NAME (helpers.classes.config_keys.ConfigKeys attribute) compact() (helpers.classes.statisticsbuilder.StatisticsBuilder.MultResProperties method) (helpers.classes.statisticsbuilder.StatisticsBuilder.RequiredProperties method) (helpers.classes.statisticsbuilder.StatisticsBuilder.TypeCountProperties method) compute_file() (in module analysis) compute_files() (in module analysis) compute_global_stats() (helpers.mongohelper.Client method) config() (in module webapp.routes) ConfigKeys (class in helpers.classes.config_keys) CONNECTION (helpers.classes.errors.ScrapingErrors attribute) CONTENTTYPE (helpers.classes.errors.ScrapingErrors attribute) cprint() (in module helpers.colors) create_bubblechart_graphic_iobytes() (in module helpers.charthelper)

create_config() (in module webapp.routes) create_filesize_histogram_iobytes() (in module helpers.charthelper) create_local_config() (in module webapp.config_setup) create_message() (in module helpers.commons) create_object() (in module helpers.nodeextractor) create_object_type() (in module helpers.nodeextractor) create_property_frequency_graphic_iobytes() (in module helpers.charthelper) create_tree_level_histogram_iobytes() (in module helpers.charthelper) create_types_graphic_iobytes() (in module helpers.charthelper) cstring() (in module helpers.colors) cwd (tests.test_array.TestStatistics attribute) (tests.test_extraction.TestExtractionGlobal attribute) (tests.test_geojson.TestStatistics attribute) (tests.test_globals.TestStatistics attribute) (tests.test_statistics.TestStatistics attribute) CYAN (helpers.colors.TColorsBackground attribute) (helpers.colors.TColorsForeground attribute)

D

DARKGREY (helpers.colors.TColorsForeground attribute) DELAY_SECONDS (helpers.classes.config_keys.ConfigKeys attribute) delete_repository() (helpers.mongohelper.Client method) (in module scraper) (in module webapp.routes) distribute_files() (in module main_server) DOWNLOAD (helpers.classes.job.JobType attribute) DOWNLOAD_DURATION (helpers.mongohelper.DurationType attribute)

download_file() (in module helpers.commons) download_files() (in module scraper) download_repository() (in module scraper) (in module webapp.routes) DOWNLOADED (helpers.classes.repositorystatus.RepositoryStatus attribute) DOWNLOADED_IR (helpers.classes.repositorystatus.RepositoryStatus attribute) DOWNLOADING (helpers.classes.repositorystatus.RepositoryStatus attribute) DownloadJob (class in helpers.classes.job) DurationType (class in helpers.mongohelper)

E

EMPTY_STRING (helpers.classes.nodetype.NodeType attribute) (helpers.mongohelper.AbuseType attribute) EMPTY_STRING_COUNT (helpers.mongohelper.Properties attribute)

end_of_object() (helpers.classes.arrayproperties.ArrayProperties method) Errors (class in helpers.classes.errors) execute_function_periodically() (in module helpers.commons)

F

fetch_analyzed_files_from_repository() (helpers.mongohelper.Client method) fetch_basic_repository_information() (helpers.mongohelper.Client method) FileStatus (class in helpers.classes.filestatus)

FILETOOLARGE (helpers.classes.errors.ScrapingErrors attribute) FINISHED (helpers.classes.filestatus.FileStatus attribute) format_bytes() (in module helpers.charthelper) full_analysis() (helpers.classes.statisticsbuilder.StatisticsBuilder method)

G

get_address() (helpers.classes.node.Node method) get_advanced_repository_statistics() (helpers.mongohelper.Client method) (in module webapp.routes) get_allof() (helpers.classes.statisticsbuilder.StatisticsBuilder.MultResProperties method) get_amount_empty() (helpers.classes.arrayproperties.ArrayProperties method) get_amount_mixed() (helpers.classes.arrayproperties.ArrayProperties method) get_amount_nested() (helpers.classes.arrayproperties.ArrayProperties method) get_amount_object() (helpers.classes.arrayproperties.ArrayProperties method) get_amount_of_abusive_booleans() (helpers.classes.statisticsbuilder.StatisticsBuilder.TypeCountProperties method) get_amount_of_abusive_numbers() (helpers.classes.statisticsbuilder.StatisticsBuilder.TypeCountProperties method) get_amount_of_analyzed_documents() (helpers.mongohelper.Client method) get_amount_of_arrays() (helpers.classes.statisticsbuilder.StatisticsBuilder.TypeCountProperties method) get_amount_of_booleans() (helpers.classes.statisticsbuilder.StatisticsBuilder.TypeCountProperties method) get_amount_of_downloaded_documents() (helpers.mongohelper.Client method) get_amount_of_empty_strings() (helpers.classes.statisticsbuilder.StatisticsBuilder.TypeCountProperties method) get_amount_of_errors() (helpers.mongohelper.Client method) get_amount_of_integers() (helpers.classes.statisticsbuilder.StatisticsBuilder.TypeCountProperties method) get_amount_of_non_empty_strings() (helpers.classes.statisticsbuilder.StatisticsBuilder.TypeCountProperties method) get_amount_of_nulls() (helpers.classes.statisticsbuilder.StatisticsBuilder.TypeCountProperties method) get_amount_of_numbers() (helpers.classes.statisticsbuilder.StatisticsBuilder.TypeCountProperties method) get_amount_of_objects() (helpers.classes.statisticsbuilder.StatisticsBuilder.TypeCountProperties method) get_amount_of_scraped_links() (helpers.mongohelper.Client method) get_amount_of_strings() (helpers.classes.statisticsbuilder.StatisticsBuilder.TypeCountProperties method) get_amount_only_array() (helpers.classes.arrayproperties.ArrayProperties method) get_amount_simple() (helpers.classes.arrayproperties.ArrayProperties method) get_amounts() (helpers.classes.statisticsbuilder.StatisticsBuilder.TypeCountProperties method) get_anyof() (helpers.classes.statisticsbuilder.StatisticsBuilder.MultResProperties method) get_available_configs() (helpers.mongohelper.Client method) get_basic_repository_information() (in module webapp.routes) get_basic_repository_statistics() (helpers.mongohelper.Client method) (in module webapp.routes) get_basic_statistics() (in module webapp.routes) get_bubblechart_provenance() (helpers.mongohelper.Client method) get_bubblechart_values() (helpers.mongohelper.Client method) (in module webapp.routes) get_children_counts() (helpers.mongohelper.Client method) get_config() (helpers.mongohelper.Client method) get_config_name() (in module webapp.config_setup) get_connection_tuple() (helpers.classes.node.Node method) get_data_type_distribution() (in module webapp.routes) get_descriptor() (helpers.classes.job.Job method) get_document_filesizes() (helpers.mongohelper.Client method) get_document_filesizes_poweroftwo() (helpers.mongohelper.Client method) get_documents_with_string_abuse() (helpers.mongohelper.Client method) (in module webapp.routes)

get_downloaded_files() (helpers.mongohelper.Client method) get_error_context_counts() (helpers.mongohelper.Client method) (in module webapp.routes) get_error_type_counts() (helpers.mongohelper.Client method) (in module webapp.routes) get_failed_downloads() (helpers.mongohelper.Client method) (in module webapp.routes) get_files() (helpers.classes.node.Node method) get_files_downloaded_by_node() (helpers.mongohelper.Client method) get_filesize_histogram() (in module webapp.routes) get_geojson_documents() (helpers.mongohelper.Client method) (in module webapp.routes) get_image_bubblechart() (in module webapp.routes) get_image_filesizes() (in module webapp.routes) get_image_property_frequencies() (in module webapp.routes) get_image_tree_levels() (in module webapp.routes) get_image_type_distribution() (in module webapp.routes) get_node() (helpers.mongohelper.Client method) (in module webapp.routes) get_node_status() (helpers.mongohelper.Client method) (in module webapp.routes) get_node_statuses() (helpers.mongohelper.Client method) (in module webapp.routes) get_node_statuses_all() (helpers.mongohelper.Client method) get_nodes() (helpers.mongohelper.Client method) get_oneof() (helpers.classes.statisticsbuilder.StatisticsBuilder.MultResProperties method) get_optional_and_required_properties() (in module webapp.routes) get_package_count() (in module scraper) get_port() (helpers.classes.node.Node method) get_property_sum() (helpers.mongohelper.Client method) get_provenance_documents() (in module webapp.routes) get_provenance_file() (in module webapp.routes) get_repositories() (helpers.mongohelper.Client method) (in module webapp.routes) get_repository_url() (helpers.mongohelper.Client method) get_results() (helpers.classes.arrayproperties.ArrayProperties method) get_scraped_links() (helpers.mongohelper.Client method) get_scraped_links_for_repository() (in module webapp.routes) get_semaphore() (helpers.classes.node.Node method) get_status() (in module webapp.routes) get_storage_directory() (helpers.classes.node.Node method) get_string_abuse_counts() (helpers.mongohelper.Client method) get_timestamp() (helpers.classes.job.Job method) get_uuid() (helpers.classes.node.Node method) GREEN (helpers.colors.TColorsBackground attribute) (helpers.colors.TColorsForeground attribute)

H

helpers module helpers.charthelper module helpers.classes module helpers.classes.arrayproperties module helpers.classes.config_keys module helpers.classes.errors module helpers.classes.exceptions module helpers.classes.filestatus module helpers.classes.job module

helpers.classes.node module helpers.classes.nodetype module helpers.classes.repositorystatus module helpers.classes.statisticsbuilder module helpers.colors module helpers.commons module helpers.confighelper module helpers.mongohelper module helpers.nodeextractor module

I

index() (in module webapp.routes) infer_type() (in module helpers.nodeextractor) INTEGER (helpers.classes.nodetype.NodeType attribute)

INTEGER_COUNT (helpers.mongohelper.Properties attribute) InvalidConfigurationError is_enabled() (helpers.classes.node.Node method) is_in_array() (helpers.classes.arrayproperties.ArrayProperties method)

J

Job (class in helpers.classes.job)

JobType (class in helpers.classes.job)

K

knows_uuid() (helpers.classes.node.Node method)

L

leaf_name() (in module helpers.nodeextractor) LIGHTBLUE (helpers.colors.TColorsForeground attribute) LIGHTCYAN (helpers.colors.TColorsForeground attribute) LIGHTGREEN (helpers.colors.TColorsForeground attribute)

LIGHTGREY (helpers.colors.TColorsBackground attribute) (helpers.colors.TColorsForeground attribute) LIGHTRED (helpers.colors.TColorsForeground attribute) listen() (in module main_server) load() (in module helpers.confighelper)

M

main_server module MASTER_ADDRESS (helpers.classes.config_keys.ConfigKeys attribute) MASTER_PORT (helpers.classes.config_keys.ConfigKeys attribute) MAX_FILESIZE (helpers.classes.config_keys.ConfigKeys attribute) MAX_LINKS_DOWNLOAD (helpers.classes.config_keys.ConfigKeys attribute) MAX_LINKS_FETCHING (helpers.classes.config_keys.ConfigKeys attribute) MAX_THREADS (helpers.classes.config_keys.ConfigKeys attribute) MIMETYPE (helpers.classes.errors.ScrapingErrors attribute) module analysis helpers helpers.charthelper helpers.classes helpers.classes.arrayproperties helpers.classes.config_keys helpers.classes.errors helpers.classes.exceptions helpers.classes.filestatus helpers.classes.job helpers.classes.node helpers.classes.nodetype helpers.classes.repositorystatus helpers.classes.statisticsbuilder helpers.colors helpers.commons helpers.confighelper helpers.mongohelper helpers.nodeextractor main_server run_webapp scraper tests tests.test_array tests.test_basis tests.test_extraction tests.test_geojson tests.test_globals tests.test_statistics webapp webapp.config_setup webapp.routes

N

nest() (helpers.classes.arrayproperties.ArrayProperties method) new_element_parsed() (in module helpers.nodeextractor) Node (class in helpers.classes.node) NODE_TYPES (helpers.classes.statisticsbuilder.StatisticsBuilder attribute) NodeType (class in helpers.classes.nodetype)

NON_EMPTY_STRING (helpers.classes.nodetype.NodeType attribute) NULL (helpers.classes.filestatus.FileStatus attribute) (helpers.classes.nodetype.NodeType attribute) NULL_COUNT (helpers.mongohelper.Properties attribute) NUMBER (helpers.classes.nodetype.NodeType attribute) NUMBER_COUNT (helpers.mongohelper.Properties attribute)

O

OBJECT (helpers.classes.nodetype.NodeType attribute) OBJECT_COUNT (helpers.mongohelper.Properties attribute) oprint() (in module helpers.commons) (in module main_server)

OPTIONAL_PROPERTIES (helpers.mongohelper.Properties attribute) ORANGE (helpers.colors.TColorsBackground attribute) (helpers.colors.TColorsForeground attribute)

P

parent_name() (in module helpers.nodeextractor) parse_document() (in module helpers.nodeextractor) PARSING (helpers.classes.errors.ScrapingErrors attribute) parsing_done() (in module helpers.nodeextractor) parsing_init() (in module helpers.nodeextractor) passed_time() (helpers.classes.job.Job method) PINK (helpers.colors.TColorsForeground attribute) print_config() (in module helpers.confighelper)

proceed_analyzed_message() (in module main_server) proceed_downloaded_message() (in module main_server) proceed_status_message() (in module main_server) Properties (class in helpers.mongohelper) PROPERTY (helpers.classes.nodetype.NodeType attribute) PROTOCOL (helpers.classes.errors.ScrapingErrors attribute) PURPLE (helpers.colors.TColorsBackground attribute) (helpers.colors.TColorsForeground attribute)

R

read_config() (in module helpers.confighelper) read_local_config() (helpers.mongohelper.Client method) recvall() (in module helpers.commons) RED (helpers.colors.TColorsBackground attribute) (helpers.colors.TColorsForeground attribute) register_sent_uuid() (helpers.classes.node.Node method) remove_file() (helpers.classes.node.Node method) remove_node() (helpers.mongohelper.Client method) (in module webapp.routes) repositories() (in module webapp.routes)

repositories_add() (in module webapp.routes) repository() (in module webapp.routes) RepositoryStatus (class in helpers.classes.repositorystatus) req_analyze_co() (helpers.classes.statisticsbuilder.StatisticsBuilder method) request_statuses() (in module main_server) REQUIRED_PROPERTIES (helpers.mongohelper.Properties attribute) RESET (helpers.colors.TStyles attribute) reset_plot() (in module helpers.charthelper) result_page() (in module webapp.routes) run_webapp module

S

scrape_chunk() (in module scraper) scrape_urls() (in module scraper) SCRAPED (helpers.classes.repositorystatus.RepositoryStatus attribute) scraper module SCRAPING (helpers.classes.repositorystatus.RepositoryStatus attribute) ScrapingErrors (class in helpers.classes.errors) send_ids() (in module main_server) send_ids_broadcast() (in module main_server) set_repository_status() (helpers.mongohelper.Client method) setUp() (tests.test_array.TestStatistics method) (tests.test_extraction.TestExtractionGlobal method) (tests.test_extraction.TestExtractionUnit method) (tests.test_geojson.TestStatistics method) (tests.test_globals.TestStatistics method) (tests.test_statistics.TestStatistics method) shutdown() (in module webapp.routes) SKIP_RESOURCES (helpers.classes.config_keys.ConfigKeys attribute)

SSL (helpers.classes.errors.ScrapingErrors attribute) start() (in module main_server) start_scraping() (in module webapp.routes) StatisticsBuilder (class in helpers.classes.statisticsbuilder) StatisticsBuilder.MultResProperties (class in helpers.classes.statisticsbuilder) StatisticsBuilder.RequiredProperties (class in helpers.classes.statisticsbuilder) StatisticsBuilder.TypeCountProperties (class in helpers.classes.statisticsbuilder) STATUSCODE (helpers.classes.errors.ScrapingErrors attribute) STORAGE_DIRECTORY (helpers.classes.config_keys.ConfigKeys attribute) store_download_error() (helpers.mongohelper.Client method) store_duration() (helpers.mongohelper.Client method) store_full_document_results() (helpers.mongohelper.Client method) store_new_file_information() (helpers.mongohelper.Client method) store_node_for_download() (helpers.mongohelper.Client method) store_scraping_error() (helpers.mongohelper.Client method) STRIKETHROUGH (helpers.colors.TStyles attribute) STRING (helpers.classes.nodetype.NodeType attribute) STRING_COUNT (helpers.mongohelper.Properties attribute)

T

TColorsBackground (class in helpers.colors) TColorsForeground (class in helpers.colors) tearDown() (tests.test_array.TestStatistics method) (tests.test_extraction.TestExtractionGlobal method) (tests.test_extraction.TestExtractionUnit method) (tests.test_geojson.TestStatistics method) (tests.test_globals.TestStatistics method) (tests.test_statistics.TestStatistics method) test_act_leaf() (tests.test_extraction.TestExtractionUnit method) test_act_map_key() (tests.test_extraction.TestExtractionUnit method) test_act_start_array() (tests.test_extraction.TestExtractionUnit method) test_act_start_map() (tests.test_extraction.TestExtractionUnit method) test_all_arrays() (tests.test_array.TestStatistics method) (tests.test_statistics.TestStatistics method) test_array() (tests.test_array.TestStatistics method) (tests.test_globals.TestStatistics method) (tests.test_statistics.TestStatistics method) test_array_dif_types() (tests.test_array.TestStatistics method) (tests.test_globals.TestStatistics method) (tests.test_statistics.TestStatistics method) test_create_object() (tests.test_extraction.TestExtractionUnit method) test_create_object_type() (tests.test_extraction.TestExtractionUnit method) test_geojson() (tests.test_geojson.TestStatistics method) test_infer() (tests.test_extraction.TestExtractionUnit method) test_leaf_name() (tests.test_extraction.TestExtractionUnit method) test_lib_is_working() (tests.test_basis.TestUnittestLibrary method) test_nested_arrays() (tests.test_globals.TestStatistics method) (tests.test_statistics.TestStatistics method) test_nested_objects() (tests.test_globals.TestStatistics method) (tests.test_statistics.TestStatistics method) test_object() (tests.test_globals.TestStatistics method) (tests.test_statistics.TestStatistics method) test_parent_name() (tests.test_extraction.TestExtractionUnit method) test_properties() (tests.test_globals.TestStatistics method) (tests.test_statistics.TestStatistics method)

test_redundancy() (tests.test_globals.TestStatistics method) (tests.test_statistics.TestStatistics method) test_simple() (tests.test_array.TestStatistics method) (tests.test_extraction.TestExtractionGlobal method) (tests.test_globals.TestStatistics method) (tests.test_statistics.TestStatistics method) test_simple_example() (tests.test_globals.TestStatistics method) (tests.test_statistics.TestStatistics method) TestExtractionGlobal (class in tests.test_extraction) TestExtractionUnit (class in tests.test_extraction) tests module tests.test_array module tests.test_basis module tests.test_extraction module tests.test_geojson module tests.test_globals module tests.test_statistics module TestStatistics (class in tests.test_array) (class in tests.test_geojson) (class in tests.test_globals) (class in tests.test_statistics) TestUnittestLibrary (class in tests.test_basis) TIMEOUT (helpers.classes.config_keys.ConfigKeys attribute) (helpers.classes.errors.ScrapingErrors attribute) toggle() (helpers.classes.node.Node method) toggle_all_nodes() (helpers.mongohelper.Client method) toggle_node() (helpers.mongohelper.Client method) (in module webapp.routes) TStyles (class in helpers.colors)

U

un_nest() (helpers.classes.arrayproperties.ArrayProperties method) UNDERLINE (helpers.colors.TStyles attribute) UNKNOWN (helpers.classes.errors.AnalysisErrors attribute) (helpers.classes.errors.ScrapingErrors attribute) UNNAMED_OBJECT (helpers.classes.nodetype.NodeType attribute)

update_config() (helpers.mongohelper.Client method) update_config_dict() (helpers.mongohelper.Client method) update_config_param() (in module webapp.config_setup) UPDATE_NODE_INTERVAL (helpers.classes.config_keys.ConfigKeys attribute) update_node_status() (helpers.mongohelper.Client method)

V

VOID (helpers.classes.nodetype.NodeType attribute)

W

webapp module webapp.config_setup module

webapp.routes module WIP (helpers.classes.filestatus.FileStatus attribute)

Y

YELLOW (helpers.colors.TColorsForeground attribute)