diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..105ce2d --- /dev/null +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..1dc95af --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..aeb7613 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/untitled.iml b/.idea/untitled.iml new file mode 100644 index 0000000..c9ea02d --- /dev/null +++ b/.idea/untitled.iml @@ -0,0 +1,11 @@ + + + + + + + + + + \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 0000000..7a48aad --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,290 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1568264130700 + + + + + + + + file://$PROJECT_DIR$/Lesson2.py + 198 + + + file://$PROJECT_DIR$/Lesson2.py + 187 + + + file://$PROJECT_DIR$/Lesson2.py + 74 + + + file://$PROJECT_DIR$/jobparser/spiders/instagram.py + 88 + + + file://$PROJECT_DIR$/jobparser/spiders/instagram.py + 89 + + + file://$PROJECT_DIR$/jobparser/spiders/instagram.py + 90 + + + file://$PROJECT_DIR$/jobparser/spiders/superjob.py + 29 + + + file://$PROJECT_DIR$/jobparser/spiders/avito.py + 12 + + + file://$PROJECT_DIR$/jobparser/spiders/avito.py + 11 + + + file://$PROJECT_DIR$/jobparser/spiders/avito.py + 13 + + + file://$PROJECT_DIR$/Lesson7.py + 5 + + + file://$PROJECT_DIR$/jobparser/spiders/zillow.py + 27 + + + file://$PROJECT_DIR$/jobparser/spiders/zillow.py + 23 + + + file://$PROJECT_DIR$/jobparser/spiders/zillow.py + 24 + + + file://$PROJECT_DIR$/jobparser/spiders/zillow.py + 36 + + + file://$PROJECT_DIR$/jobparser/spiders/zillow.py + 37 + + + file://$PROJECT_DIR$/jobparser/spiders/zillow.py + 33 + + + file://$PROJECT_DIR$/jobparser/spiders/fb.py + 50 + + + file://$PROJECT_DIR$/jobparser/spiders/fb.py + 49 + + + file://$PROJECT_DIR$/jobparser/spiders/fb.py + 62 + + + file://$PROJECT_DIR$/jobparser/spiders/fb.py + 29 + + + file://$PROJECT_DIR$/jobparser/spiders/fb.py + 63 + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Lesson2.py b/Lesson2.py new file mode 100644 index 0000000..53214b1 --- /dev/null +++ b/Lesson2.py @@ -0,0 +1,200 @@ +from bs4 import BeautifulSoup +#import lxml +import requests +import json +import time +import random +import re +from pymongo import MongoClient + + +class findJob: + USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:69.0) Gecko/20100101 Firefox/69.0' + + def __init__(self): # , spec, page_start, pages, sleep_min, sleep_max): + self.spec_list = False + self.spec_to_parse = False + self.parse_sleep_min = 10 + self.parse_sleep_max = 20 + self.is_hh = False + self.is_sj = False + self.hh_page_start = 0 + self.sj_page_start = 0 + self.sleep_min = 1 + self.sleep_max = 5 + self.vak_array = [] + + def get_init_data(self): + self.chooose_spec() + self.is_hh = int(input("Парсить HH? 0 - нет; 1 - да")) + if self.is_hh == 1: + self.hh_page_start = int(input("Номер страницы с которой начать парсить HH. 0 - с первой страницы")) + if self.hh_page_start < 0: self.hh_page_to_parse = 0 + self.hh_page_to_parse = int(input("Сколько страниц парсить. Минимум 1")) + if self.hh_page_to_parse < 1: self.hh_page_to_parse = 1 + else: + self.is_hh = 0 + + self.is_sj = int(input("Парсить SJ? 0 - нет; 1 - да")) + if self.is_sj == 1: + self.sj_page_start = int(input("Номер страницы с которой начать парсить SJ. 0 - с первой страницы")) + if self.sj_page_start < 0: self.sj_page_start = 0 + self.sj_page_to_parse = int(input("Сколько страниц парсить. Минимум 1")) + if self.sj_page_to_parse < 1: self.sj_page_to_parse = 1 + else: + self.is_sj = 0 + + def check_init_data(self): + if self.is_hh: + print(f"Будет осуществляться парсинг HH") + print(f" Парсинг начнётся со страницы {self.hh_page_start}") + print(f" Количество обрабатываемых страниц {self.hh_page_to_parse}") + + if self.is_sj: + print(f"Будет осуществляться парсинг HH") + print(f" Парсинг начнётся со страницы {self.sj_page_start}") + print(f" Количество обрабатываемых страниц {self.sj_page_to_parse}") + + def parse_start(self): + if self.is_hh: + for i in range(self.hh_page_start, self.hh_page_start+self.hh_page_to_parse): + self.get_hh_spec_page(i) + + if self.is_sj: + for i in range(self.sj_page_start, self.sj_page_start + self.sj_page_to_parse): + self.get_sj_spec_page(i) + + def save_result_to_db(self): + mongo_url = 'mongodb://localhost:27017/' + client = MongoClient(mongo_url) + database = client.lesson2 + collection = database.lessonnnn2 + + x = collection.insert_many(self.vak_array) + + print(x.inserted_ids) + + + def get_spec_list(self): + url = "https://www.superjob.ru/vakansii/" + request = requests.get(url, headers={'User-Agent': self.USER_AGENT}) + soup = BeautifulSoup(request.text, "lxml") + body = soup.html.body + vakansii = body.findAll("a", attrs={"href": re.compile("/vakansii/(.*?).html")}) + result = [] + for i in range(len(vakansii)): + result.append({"vak_name": vakansii[i].text, + "vak_link": vakansii[i]['href']}) + + self.spec_list = result + + def chooose_spec(self): + print(f"Список доступных профессий") + for i in range(len(self.spec_list)): + print(f"[{i}] - {self.spec_list[i]['vak_name']}") + + self.spec_to_parse = int(input(f"Введите число соответствующее профессии:")) + + def get_result_array(self): + return self.vak_array + + def get_hh_spec_page(self, page=False, base_url="https://ekaterinburg.hh.ru/search/vacancy?clusters=true&enable_snippets=true&no_magic=true&area=113&from=cluster_area&showClusters=true&text="): + final_url = f"{base_url}{self.spec_list[self.spec_to_parse]['vak_name'].replace(' ', '+')}{f'&page={page-1}' if page>1 else ''}" + #print(final_url) + response = requests.get(final_url, headers={'User-Agent': self.USER_AGENT}) + #print(response) + soup = BeautifulSoup(response.text, "lxml") + body = soup.html.body + #print(body) + vakansii = body.findAll("div", attrs={"class": re.compile("vacancy-serp-item__row vacancy-serp-item__row_header")}) + #print(vakansii) + + for i in range (len(vakansii)): + #print(vakansii[i]) + vak_data = vakansii[i].find("a") + #print(vak_data) + vak_name = vak_data.text + #print(vak_name) + vak_link = vak_data.get('href') + vak_price_min = "Не указано" + vak_price_max = "Не указано" + vak_price = vakansii[i].find("div", attrs={"class": "vacancy-serp-item__compensation"}) + if vak_price != None: + temp_price = vak_price.text.replace(" руб.", "") + temp_price = temp_price.replace(u"\xa0", "") + if "от" in temp_price: + temp_price = temp_price.replace("от", "") + vak_price_min = temp_price + elif "до" in temp_price: + temp_price = temp_price.replace("до", "") + vak_price_max = temp_price + else: + temp_price=temp_price.split("-") + vak_price_min=temp_price[0] + vak_price_max=temp_price[1] + + print(f"Вакансия: {vak_name}") + print(f"Максимальная ЗП: {vak_price_max}") + print(f"Минимальная ЗП: {vak_price_min}") + print(f"Ссылка: {vak_link}") + print(f"Источник: {vak_link.split('/')[2]}") + print("=========================================") + self.vak_array.append({"vac_name": vak_name, + "vac_price_max": vak_price_max, + "vac_price_min": vak_price_min, + "vac_link": vak_link, + "vac_source": vak_link.split('/')[2]}) + + time.sleep(random.randint(self.sleep_min, self.sleep_max)) + + + + def get_sj_spec_page(self, page=False, base_url="https://www.superjob.ru"): + final_url = f"{base_url}{self.spec_list[self.spec_to_parse]['vak_link']}{f'?page={page}' if page>1 else ''}" + response = requests.get(final_url, headers={'User-Agent': self.USER_AGENT}) + soup = BeautifulSoup(response.text, "lxml") + body = soup.html.body + vakansii = body.findAll("script", attrs={"type": "application/ld+json"}) + json_vak = json.loads(vakansii[1].text) + + for row in json_vak["itemListElement"]: + response2 = requests.get(row["url"], headers={'User-Agent': self.USER_AGENT}) + soup2 = BeautifulSoup(response2.text, "lxml") + cur_vakansija = soup2.html.body.findAll("script", attrs={"type": "application/ld+json"}) + json_cur_vakansija = json.loads(cur_vakansija[1].text) + print(f"Вакансия: {json_cur_vakansija['title']}") + + min_price = "Не указано" + max_price = "Не указано" + if json_cur_vakansija.get('baseSalary', False): + if json_cur_vakansija['baseSalary'].get('value', False): + if json_cur_vakansija['baseSalary']['value'].get('minValue', False): + min_price = json_cur_vakansija['baseSalary']['value']['minValue'] + if json_cur_vakansija['baseSalary']['value'].get('maxValue', False): + max_price = json_cur_vakansija['baseSalary']['value']['maxValue'] + + print(f"Максимальная ЗП: {max_price}") + print(f"Минимальная ЗП: {min_price}") + print(f"Ссылка: {json_cur_vakansija['url']}") + print(f"Источник: {json_cur_vakansija['url'].split('/')[2]}") + print("=========================================") + + self.vak_array.append({"vac_name": json_cur_vakansija['title'], + "vac_price_max": max_price, + "vac_price_min": min_price, + "vac_link": json_cur_vakansija['url'], + "vac_source": json_cur_vakansija['url'].split('/')[2]}) + + time.sleep(random.randint(self.sleep_min, self.sleep_max)) + + +if __name__ == "__main__": + parse_jobs = findJob() + parse_jobs.get_spec_list() + parse_jobs.get_init_data() + parse_jobs.parse_start() + parse_jobs.save_result_to_db() + print(1) + +print(4) +exit(0) diff --git a/Lesson3.py b/Lesson3.py new file mode 100644 index 0000000..de7b309 --- /dev/null +++ b/Lesson3.py @@ -0,0 +1,3 @@ +import requests +import wget + diff --git a/Lesson4.py b/Lesson4.py new file mode 100644 index 0000000..e69de29 diff --git a/Lesson5.py b/Lesson5.py new file mode 100644 index 0000000..e69de29 diff --git a/Lesson7.py b/Lesson7.py new file mode 100644 index 0000000..1a8945c --- /dev/null +++ b/Lesson7.py @@ -0,0 +1,6 @@ +# from selenium import webdriver +# from selenium.webdriver.common.keys import Keys +# fox_driver = webdriver.Firefox() +# fox_driver.get('https://geekbrains.ru/') +# +# pass \ No newline at end of file diff --git a/database/__init__.py b/database/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/database/base.py b/database/base.py new file mode 100644 index 0000000..d919092 --- /dev/null +++ b/database/base.py @@ -0,0 +1,26 @@ +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from sqlalchemy_utils import database_exists, create_database +from database.models import Vacancy, Base + +class VacancyDB: + + def __init__(self, url): + self.url = url + pass + + + def add_salery(self, db_item): + self.engine = create_engine(self.url) + Base.metadata.create_all(self.engine) + + db_session = sessionmaker(bind=self.engine) + session = db_session() + session.add(db_item) + try: + session.commit() + except Exception as e: + print(e) + session.close() + + diff --git a/database/models.py b/database/models.py new file mode 100644 index 0000000..9baeb92 --- /dev/null +++ b/database/models.py @@ -0,0 +1,19 @@ +from sqlalchemy import Table, Column, String, Integer + +from sqlalchemy.ext.declarative import declarative_base + +Base = declarative_base() + +class Vacancy(Base): + __tablename__ = 'avito' + id = Column(Integer, primary_key=True, unique=True, autoincrement=True) + spider = Column(String, nullable=True) + name = Column(String, nullable=True) + salary = Column(String, nullable=True) + company = Column(String, nullable=True) + + def __init__(self, **kwargs): + self.name = kwargs.get('name') + self.spider = kwargs.get('spider') + self.salary = kwargs.get('salary') + # self.company = kwargs.get('company') \ No newline at end of file diff --git a/geckodriver.log b/geckodriver.log new file mode 100644 index 0000000..247a581 --- /dev/null +++ b/geckodriver.log @@ -0,0 +1,588 @@ +1570110375072 mozrunner::runner INFO Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "-marionette" "-foreground" "-no-remote" "-profile" "C:\\Users\\Admin\\AppData\\Local\\Temp\\rust_mozprofileiZwfAT" +1570110377174 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: mozillaAddons +1570110377176 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: telemetry +1570110377177 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: resource://pdf.js/ +1570110377178 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570110384391 Marionette INFO Listening on port 56963 +1570110384442 Marionette WARN TLS certificate errors will be ignored for this session +[Parent 3772, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Parent 3772, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Parent 3772, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 3564, Chrome_ChildThread] WARNING: pipe error: 232: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 502 +[Child 3564, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 3564, Ch[Child 9684, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 9684, Chrome_Ch[Parent 3772, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 6124, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 6124, Chrome_ChildThre1570110398999 Marionette INFO Stopped listening on port 56963 +1570165048626 mozrunner::runner INFO Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "-marionette" "-foreground" "-no-remote" "-profile" "C:\\Users\\Admin\\AppData\\Local\\Temp\\rust_mozprofile36eDD6" +1570165053836 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: mozillaAddons +1570165053837 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: telemetry +1570165053838 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: resource://pdf.js/ +1570165053838 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570165062662 Marionette INFO Listening on port 49938 +1570165062938 Marionette WARN TLS certificate errors will be ignored for this session +[Parent 5600, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Parent 5600, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 3536, Chro[Parent 5600, Geckme_o_ICOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 3268, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 3268, [Parent 5600, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 320, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 320, Chrome_Ch1570165081999 Marionette INFO Stopped listening on port 49938 +1570337877220 mozrunner::runner INFO Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "-marionette" "-foreground" "-no-remote" "-profile" "C:\\Users\\Admin\\AppData\\Local\\Temp\\rust_mozprofileWKYb24" +1570337879728 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: mozillaAddons +1570337879729 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: telemetry +1570337879730 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: resource://pdf.js/ +1570337879730 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570337885699 Marionette INFO Listening on port 51237 +1570337886086 Marionette WARN TLS certificate errors will be ignored for this session +[Parent 3116, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Parent 3116, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Parent 3116, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 2784, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 2784, Chrome_ChildThread] WARNING: pipe error: 109: fil[Parent 3116, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 2332, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 2332, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +1570337930208 Marionette INFO Stopped listening on port 51237 +tening on port 51339 +1570337927445 Marionette WARN TLS certificate errors will be ignored for this session +[Parent 6988, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +JavaScript warning: https://data.gov.ru/sites/default/files/js/js_LuIt6PC5SOWN-W_wGTtNFQ1LUC3XNV7FxZ4m2WwGeo8.js, line 3184: unreachable code after return statement +JavaScript warning: https://data.gov.ru/sites/default/files/js/js_LuIt6PC5SOWN-W_wGTtNFQ1LUC3XNV7FxZ4m2WwGeo8.js, line 3184: unreachable code after return statement +[Parent 6988, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +1570338115956 mozrunner::runner INFO Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "-marionette" "-foreground" "-no-remote" "-profile" "C:\\Users\\Admin\\AppData\\Local\\Temp\\rust_mozprofileG6crXZ" +1570338117688 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: mozillaAddons +1570338117689 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: telemetry +1570338117690 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: resource://pdf.js/ +1570338117690 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570338122989 Marionette INFO Listening on port 51451 +1570338123119 Marionette WARN TLS certificate errors will be ignored for this session +1570344216496 mozrunner::runner INFO Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "-marionette" "-foreground" "-no-remote" "-profile" "C:\\Users\\Admin\\AppData\\Local\\Temp\\rust_mozprofileUDMigC" +1570344218278 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: mozillaAddons +1570344218279 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: telemetry +1570344218280 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: resource://pdf.js/ +1570344218280 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570344225677 Marionette INFO Listening on port 51889 +1570344225874 Marionette WARN TLS certificate errors will be ignored for this session +[Parent 8932, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Parent 8932, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 3872, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 3872, Chrome_C[Parent 8932, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 6404, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 6404, Chrome_ChildThread] WARNING: pipe1570344247252 Marionette INFO Stopped listening on port 51889 +1570344268697 mozrunner::runner INFO Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "-marionette" "-foreground" "-no-remote" "-profile" "C:\\Users\\Admin\\AppData\\Local\\Temp\\rust_mozprofileK7zcC5" +1570344270498 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: mozillaAddons +1570344270499 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: telemetry +1570344270500 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: resource://pdf.js/ +1570344270500 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570344276009 Marionette INFO Listening on port 51939 +1570344276345 Marionette WARN TLS certificate errors will be ignored for this session +[Parent 1028, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 4756, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 4756, Chrome_ChildThread] WARNING: pipe er[Child 2408, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 2408, Chrome_Chil1570344479680 Marionette INFO Stopped listening on port 51939 +eenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: resource://pdf.js/ +1570344437087 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570344442810 Marionette INFO Listening on port 52000 +1570344443015 Marionette WARN TLS certificate errors will be ignored for this session +[Parent 6336, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 4028, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 4028, Chrome_ChildT[Child 3668, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 3668, Chrome_C[Parent 6336, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 6344, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 6344, Chrome1570344479863 Marionette INFO Stopped listening on port 52000 +1570344524031 mozrunner::runner INFO Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "-marionette" "-foreground" "-no-remote" "-profile" "C:\\Users\\Admin\\AppData\\Local\\Temp\\rust_mozprofile4ziWrZ" +1570344525857 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: mozillaAddons +1570344525858 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: telemetry +1570344525858 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: resource://pdf.js/ +1570344525859 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570344530801 Marionette INFO Listening on port 52045 +1570344531192 Marionette WARN TLS certificate errors will be ignored for this session +[Parent 1616, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Parent 1616, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 6272, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 6272,[Parent 1616, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +1570345315652 Marionette INFO Stopped listening on port 52045 + +###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost + +lla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570344578722 Marionette INFO Listening on port 52090 +1570344579089 Marionette WARN TLS certificate errors will be ignored for this session +[Child 4316, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 4316[Parent 7036, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 6404, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 6404, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +1570345315691 Marionette INFO Stopped listening on port 52090 +alid extension permission: resource://pdf.js/ +1570344627811 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570344633518 Marionette INFO Listening on port 52137 +1570344633691 Marionette WARN TLS certificate errors will be ignored for this session +[Child 8764, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 8764, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1[Parent 8984, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 6336, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 6336, Chrome_ChildThr[Parent 8984, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 6520, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 6520, Chrome_ChildThread] WARNING: pipe error: 109: file z:1570345315694 Marionette INFO Stopped listening on port 52137 + directory. +1570344764432 Marionette INFO Listening on port 52253 +1570344764474 Marionette WARN TLS certificate errors will be ignored for this session +[Parent 5336, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 7692, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 7692, Chrome_ChildThr[Child 9364, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 9364, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 4660, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 4660, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +1570345315690 Marionette INFO Stopped listening on port 52253 +44826480 Marionette INFO Listening on port 52298 +1570344826741 Marionette WARN TLS certificate errors will be ignored for this session +[Parent 1052, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 2120, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child [Parent 1052, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 7332, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 7332, 10180, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 10180, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +1570345315768 Marionette INFO Stopped listening on port 52298 +irectory. +1570345298123 Marionette INFO Listening on port 52396 +1570345298200 Marionette WARN TLS certificate errors will be ignored for this session +[Parent 10548, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 10944, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 1[Child 9156, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 9156, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Parent 10548, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 10716, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 10716, Chrome_ChildThread] WARNING: pipe error: 101570345315920 Marionette INFO Stopped listening on port 52396 +1570345333290 mozrunner::runner INFO Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "-marionette" "-foreground" "-no-remote" "-profile" "C:\\Users\\Admin\\AppData\\Local\\Temp\\rust_mozprofileS1lsxl" +1570345334850 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: mozillaAddons +1570345334851 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: telemetry +1570345334852 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: resource://pdf.js/ +1570345334852 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570345339979 Marionette INFO Listening on port 52440 +1570345340436 Marionette WARN TLS certificate errors will be ignored for this session +1570345412693 mozrunner::runner INFO Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "-marionette" "-foreground" "-no-remote" "-profile" "C:\\Users\\Admin\\AppData\\Local\\Temp\\rust_mozprofile80WAIB" +1570345414565 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: mozillaAddons +1570345414566 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: telemetry +1570345414567 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: resource://pdf.js/ +1570345414567 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570345421017 Marionette INFO Listening on port 52486 +1570345421452 Marionette WARN TLS certificate errors will be ignored for this session +1570345572318 mozrunner::runner INFO Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "-marionette" "-foreground" "-no-remote" "-profile" "C:\\Users\\Admin\\AppData\\Local\\Temp\\rust_mozprofileLZ9YbU" +1570345574050 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: mozillaAddons +1570345574051 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: telemetry +1570345574051 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: resource://pdf.js/ +1570345574051 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570345578112 Marionette INFO Listening on port 52591 +1570345578369 Marionette WARN TLS certificate errors will be ignored for this session +1570345626886 mozrunner::runner INFO Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "-marionette" "-foreground" "-no-remote" "-profile" "C:\\Users\\Admin\\AppData\\Local\\Temp\\rust_mozprofileyvUxQH" +1570345628744 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: mozillaAddons +1570345628745 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: telemetry +1570345628746 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: resource://pdf.js/ +1570345628746 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570345633956 Marionette INFO Listening on port 52669 +1570345634031 Marionette WARN TLS certificate errors will be ignored for this session +[Parent 4520, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Parent 4520, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 3644, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 3644, Chro[Child 9512, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 9512, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/[Parent 4520, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 10780, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 10780, Chromconsole.error: (new Error("Connection closed before committing the transaction.", "resource://gre/modules/Sqlite.jsm", 683)) +1570347256664 Marionette INFO Stopped listening on port 52669 +1570347272204 mozrunner::runner INFO Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "-marionette" "-foreground" "-no-remote" "-profile" "C:\\Users\\Admin\\AppData\\Local\\Temp\\rust_mozprofilel4rtWW" +1570347273979 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: mozillaAddons +1570347273980 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: telemetry +1570347273981 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: resource://pdf.js/ +1570347273981 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570347279739 Marionette INFO Listening on port 52891 +1570347279858 Marionette WARN TLS certificate errors will be ignored for this session +1570352402894 mozrunner::runner INFO Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "-marionette" "-foreground" "-no-remote" "-profile" "C:\\Users\\Admin\\AppData\\Local\\Temp\\rust_mozprofile4YiS28" +1570352404798 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: mozillaAddons +1570352404799 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: telemetry +1570352404800 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: resource://pdf.js/ +1570352404800 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570352411009 Marionette INFO Listening on port 53136 +1570352411167 Marionette WARN TLS certificate errors will be ignored for this session +JavaScript error: , line 0: uncaught exception: Object +1570353379724 mozrunner::runner INFO Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "-marionette" "-foreground" "-no-remote" "-profile" "C:\\Users\\Admin\\AppData\\Local\\Temp\\rust_mozprofile9rlTRS" +1570353381581 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: mozillaAddons +1570353381582 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: telemetry +1570353381583 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: resource://pdf.js/ +1570353381583 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570353386688 Marionette INFO Listening on port 53319 +1570353386878 Marionette WARN TLS certificate errors will be ignored for this session +JavaScript error: , line 0: uncaught exception: Object +[Parent 9820, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Parent 9820, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 9000, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 9000, Chrome_ChildThread] WARNING: pipe er[Child 704, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 704, Chrome_ChildThread] WARNING: pipe error: 109: file z:/console.error: (new Error("Connection closed before committing the transaction.", "resource://gre/modules/Sqlite.jsm", 683)) +1570353742063 Marionette INFO Stopped listening on port 53319 +1570353753636 mozrunner::runner INFO Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "-marionette" "-foreground" "-no-remote" "-profile" "C:\\Users\\Admin\\AppData\\Local\\Temp\\rust_mozprofileFMzCWz" +1570353754422 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: mozillaAddons +1570353754423 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: telemetry +1570353754423 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: resource://pdf.js/ +1570353754424 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570353757336 Marionette INFO Listening on port 53638 +1570353757477 Marionette WARN TLS certificate errors will be ignored for this session +JavaScript error: , line 0: uncaught exception: Object +1570452065770 mozrunner::runner INFO Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "-marionette" "-foreground" "-no-remote" "-profile" "C:\\Users\\Admin\\AppData\\Local\\Temp\\rust_mozprofilea7DkSX" +1570452067929 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: mozillaAddons +1570452067930 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: telemetry +1570452067931 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: resource://pdf.js/ +1570452067932 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570452074737 Marionette INFO Listening on port 54897 +1570452075135 Marionette WARN TLS certificate errors will be ignored for this session +[Parent 4152, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 1112, Chrome_ChildThread] WARNING: pipe error: 232: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 502 +[Child 1112, Chrome_ChildThread] WARNING: pipe error: 232: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 502 +[Child 1112, Chrome_ChildThread] WARNING: pipe error: 232: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 502 +[ +###!!! [Child][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost + +Child 1112, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 1112, Chrome_ChildThread] WARNING: pipe error: 10[Parent 4152, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 5584, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 5584, Chrome_1570452135639 Marionette INFO Stopped listening on port 54897 +1570452150852 mozrunner::runner INFO Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "-marionette" "-foreground" "-no-remote" "-profile" "C:\\Users\\Admin\\AppData\\Local\\Temp\\rust_mozprofileBiaOOf" +1570452152643 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: mozillaAddons +1570452152643 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: telemetry +1570452152644 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: resource://pdf.js/ +1570452152645 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570452157430 Marionette INFO Listening on port 55060 +1570452157505 Marionette WARN TLS certificate errors will be ignored for this session +1570452513861 mozrunner::runner INFO Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "-marionette" "-foreground" "-no-remote" "-profile" "C:\\Users\\Admin\\AppData\\Local\\Temp\\rust_mozprofilePkIAFd" +1570452515760 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: mozillaAddons +1570452515761 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: telemetry +1570452515762 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: resource://pdf.js/ +1570452515762 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570452522096 Marionette INFO Listening on port 55317 +1570452522164 Marionette WARN TLS certificate errors will be ignored for this session +1570466370454 mozrunner::runner INFO Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "-marionette" "-foreground" "-no-remote" "-profile" "C:\\Users\\Admin\\AppData\\Local\\Temp\\rust_mozprofileghO36t" +1570466372322 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: mozillaAddons +1570466372322 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: telemetry +1570466372323 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: resource://pdf.js/ +1570466372323 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570466375199 Marionette INFO Listening on port 55679 +1570466375400 Marionette WARN TLS certificate errors will be ignored for this session +1570466818448 mozrunner::runner INFO Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "-marionette" "-foreground" "-no-remote" "-profile" "C:\\Users\\Admin\\AppData\\Local\\Temp\\rust_mozprofiledul4d2" +1570466820325 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: mozillaAddons +1570466820326 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: telemetry +1570466820326 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: resource://pdf.js/ +1570466820327 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570466826945 Marionette INFO Listening on port 55848 +1570466827243 Marionette WARN TLS certificate errors will be ignored for this session +1570466917332 mozrunner::runner INFO Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "-marionette" "-foreground" "-no-remote" "-profile" "C:\\Users\\Admin\\AppData\\Local\\Temp\\rust_mozprofilemfumsx" +1570466919176 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: mozillaAddons +1570466919177 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: telemetry +1570466919177 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: resource://pdf.js/ +1570466919178 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570466925165 Marionette INFO Listening on port 56024 +1570466925600 Marionette WARN TLS certificate errors will be ignored for this session +1570467345977 mozrunner::runner INFO Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "-marionette" "-foreground" "-no-remote" "-profile" "C:\\Users\\Admin\\AppData\\Local\\Temp\\rust_mozprofileyL5SA8" +1570467347882 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: mozillaAddons +1570467347882 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: telemetry +1570467347883 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: resource://pdf.js/ +1570467347884 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570467353836 Marionette INFO Listening on port 56267 +1570467354230 Marionette WARN TLS certificate errors will be ignored for this session +1570467479468 mozrunner::runner INFO Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "-marionette" "-foreground" "-no-remote" "-profile" "C:\\Users\\Admin\\AppData\\Local\\Temp\\rust_mozprofileDLgBRl" +1570467481376 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: mozillaAddons +1570467481377 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: telemetry +1570467481378 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: resource://pdf.js/ +1570467481379 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570467487723 Marionette INFO Listening on port 56437 +1570467488222 Marionette WARN TLS certificate errors will be ignored for this session +[Parent 10088, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +JavaScript error: resource://gre/actors/BrowserElementParent.jsm, line 314: TypeError: state is undefined +JavaScript error: , line 0: NS_ERROR_UNEXPECTED: +JavaScript error: resource://gre/actors/BrowserElementParent.jsm, line 314: TypeError: state is undefined +JavaScript error: , line 0: NS_ERROR_UNEXPECTED: +[Parent 10088, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 2544, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 2544, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Parent 10088, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 10612, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 10612, C1570467596876 Marionette INFO Stopped listening on port 56437 +1570467613784 mozrunner::runner INFO Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "-marionette" "-foreground" "-no-remote" "-profile" "C:\\Users\\Admin\\AppData\\Local\\Temp\\rust_mozprofileARgUd7" +1570467615611 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: mozillaAddons +1570467615612 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: telemetry +1570467615612 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: resource://pdf.js/ +1570467615613 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570467622132 Marionette INFO Listening on port 56580 +1570467622552 Marionette WARN TLS certificate errors will be ignored for this session +1570468415395 mozrunner::runner INFO Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "-marionette" "-foreground" "-no-remote" "-profile" "C:\\Users\\Admin\\AppData\\Local\\Temp\\rust_mozprofileCAsXND" +1570468417259 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: mozillaAddons +1570468417260 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: telemetry +1570468417261 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: resource://pdf.js/ +1570468417261 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570468423358 Marionette INFO Listening on port 56803 +1570468423650 Marionette WARN TLS certificate errors will be ignored for this session +[Parent 10544, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Parent 10544, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 10076, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 10076, [Child 4396, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 4396, Chrom[Child 2124, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 2124, Chrome_ChildThread] 1570468439243 Marionette INFO Stopped listening on port 56803 +1570468899248 mozrunner::runner INFO Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "-marionette" "-foreground" "-no-remote" "-profile" "C:\\Users\\Admin\\AppData\\Local\\Temp\\rust_mozprofileefr4Wu" +1570468901085 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: mozillaAddons +1570468901086 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: telemetry +1570468901087 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: resource://pdf.js/ +1570468901088 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570468907578 Marionette INFO Listening on port 56963 +1570468908032 Marionette WARN TLS certificate errors will be ignored for this session +[Parent 10592, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Parent 10592, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 8672, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 8672, Chrom[Child 3436, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 3436, Chrom[Parent 10592, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 9628, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 9628, Chrome_C1570468914243 Marionette INFO Stopped listening on port 56963 +1570469031219 mozrunner::runner INFO Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "-marionette" "-foreground" "-no-remote" "-profile" "C:\\Users\\Admin\\AppData\\Local\\Temp\\rust_mozprofilek5iInL" +1570469033044 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: mozillaAddons +1570469033044 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: telemetry +1570469033045 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: resource://pdf.js/ +1570469033046 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570469039137 Marionette INFO Listening on port 57008 +1570469039465 Marionette WARN TLS certificate errors will be ignored for this session +1570469147015 mozrunner::runner INFO Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "-marionette" "-foreground" "-no-remote" "-profile" "C:\\Users\\Admin\\AppData\\Local\\Temp\\rust_mozprofileAoCiL4" +1570469148927 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: mozillaAddons +1570469148928 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: telemetry +1570469148929 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: resource://pdf.js/ +1570469148929 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570469155458 Marionette INFO Listening on port 57061 +1570469155783 Marionette WARN TLS certificate errors will be ignored for this session +1570469235110 mozrunner::runner INFO Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "-marionette" "-foreground" "-no-remote" "-profile" "C:\\Users\\Admin\\AppData\\Local\\Temp\\rust_mozprofilesyIIir" +1570469236928 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: mozillaAddons +1570469236928 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: telemetry +1570469236929 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: resource://pdf.js/ +1570469236930 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570469243671 Marionette INFO Listening on port 57109 +1570469243886 Marionette WARN TLS certificate errors will be ignored for this session +1570469247265 mozrunner::runner INFO Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "-marionette" "-foreground" "-no-remote" "-profile" "C:\\Users\\Admin\\AppData\\Local\\Temp\\rust_mozprofileSpUguA" +1570469249123 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: mozillaAddons +1570469249124 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: telemetry +1570469249125 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: resource://pdf.js/ +1570469249126 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570469255643 Marionette INFO Listening on port 57145 +1570469256039 Marionette WARN TLS certificate errors will be ignored for this session +1570469353551 mozrunner::runner INFO Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "-marionette" "-foreground" "-no-remote" "-profile" "C:\\Users\\Admin\\AppData\\Local\\Temp\\rust_mozprofileccm7si" +1570469355312 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: mozillaAddons +1570469355313 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: telemetry +1570469355314 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: resource://pdf.js/ +1570469355314 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570469362152 Marionette INFO Listening on port 57188 +1570469362320 Marionette WARN TLS certificate errors will be ignored for this session +JavaScript error: resource://devtools/client/framework/toolbox.js, line 1717: TypeError: currentPanel is undefined +1570469858534 mozrunner::runner INFO Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "-marionette" "-foreground" "-no-remote" "-profile" "C:\\Users\\Admin\\AppData\\Local\\Temp\\rust_mozprofile30xioe" +1570469860403 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: mozillaAddons +1570469860403 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: telemetry +1570469860405 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: resource://pdf.js/ +1570469860405 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570469867021 Marionette INFO Listening on port 57252 +1570469867309 Marionette WARN TLS certificate errors will be ignored for this session +JavaScript warning: https://static.xx.fbcdn.net/rsrc.php/v3iGIk4/yV/l/ru_RU/07lSJho0MKB.js?_nc_x=VgXmsmo3SSv, line 18: Error: WebGL warning: getContext: Disallowing antialiased backbuffers due to blacklisting. +JavaScript warning: https://static.xx.fbcdn.net/rsrc.php/v3/y8/r/2VB0VDfSp1N.js?_nc_x=VgXmsmo3SSv, line 4: Error: WebGL warning: getContext: Disallowing antialiased backbuffers due to blacklisting. +console.error: "Error while calling actor 'layout's method 'getGrids'" "node is null" +console.error: "getGrids@resource://devtools/server/actors/layout.js:501:26\ngetGrids@resource://devtools/server/actors/layout.js:506:43\nhandler@resource://devtools/shared/protocol/Actor.js:154:37\nonPacket@resource://devtools/server/main.js:1402:58\nreceiveMessage@resource://devtools/shared/transport/child-transport.js:66:16\n" +[Parent 10296, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Parent 10296, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 10088, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 10088, Chrome_ChildThread] WARNING: pipe error: 109: file z:/ta[Parent 10296, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 10584, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 10584,1570470545510 Marionette INFO Stopped listening on port 57252 +task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 10844, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 10844, Chrome_1570470545200 Marionette INFO Stopped listening on port 57289 +1570470563504 mozrunner::runner INFO Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "-marionette" "-foreground" "-no-remote" "-profile" "C:\\Users\\Admin\\AppData\\Local\\Temp\\rust_mozprofileJ5yvRs" +1570470565352 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: mozillaAddons +1570470565352 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: telemetry +1570470565353 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: resource://pdf.js/ +1570470565354 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570470571464 Marionette INFO Listening on port 57384 +1570470571772 Marionette WARN TLS certificate errors will be ignored for this session +[Parent 10640, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Parent 10640, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 11196, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 11196, Chrome_ChildThread] WARNING: pipe error: 109: fil[Parent 10640, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 9516, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 9516, Chrom1570470622269 Marionette INFO Stopped listening on port 57384 +JavaScript error: resource://gre/modules/osfile/osfile_async_front.jsm, line 426: Error: OS.File has been shut down. Rejecting post to stat +port 57420 +1570470583938 Marionette WARN TLS certificate errors will be ignored for this session +[Parent 9624, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 5308, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 5308, Chrome[Parent 9624, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 8760, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 8760, Chrome_ChildTh1570470622700 Marionette INFO Stopped listening on port 57420 +1570470643358 mozrunner::runner INFO Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "-marionette" "-foreground" "-no-remote" "-profile" "C:\\Users\\Admin\\AppData\\Local\\Temp\\rust_mozprofileUXoTn4" +1570470645205 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: mozillaAddons +1570470645206 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: telemetry +1570470645207 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: resource://pdf.js/ +1570470645207 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570470651622 Marionette INFO Listening on port 57490 +1570470652112 Marionette WARN TLS certificate errors will be ignored for this session +JavaScript warning: https://static.xx.fbcdn.net/rsrc.php/v3/y8/r/2VB0VDfSp1N.js?_nc_x=VgXmsmo3SSv, line 4: Error: WebGL warning: getContext: Disallowing antialiased backbuffers due to blacklisting. +JavaScript warning: https://static.xx.fbcdn.net/rsrc.php/v3iGIk4/yD/l/ru_RU/6_KXv7olM2l.js?_nc_x=VgXmsmo3SSv, line 18: Error: WebGL warning: getContext: Disallowing antialiased backbuffers due to blacklisting. +[Parent 3308, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +JavaScript error: https://static.xx.fbcdn.net/rsrc.php/v3iBRl4/y4/l/ru_RU/ZVuz92lEEOH.js?_nc_x=VgXmsmo3SSv, line 5: TypeError: The expression cannot be converted to return the specified type. +console.error: "Error while calling actor 'layout's method 'getGrids'" "node is null" +console.error: "getGrids@resource://devtools/server/actors/layout.js:501:26\ngetGrids@resource://devtools/server/actors/layout.js:506:43\nhandler@resource://devtools/shared/protocol/Actor.js:154:37\nonPacket@resource://devtools/server/main.js:1402:58\nreceiveMessage@resource://devtools/shared/transport/chil[Parent 8392, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 8312, Chrome_ChildThread] WARN[PareINGnt 83: pipe error:92, Gecko_IOTh 109: fileread] z: WARN/task_156ING:8728986/bu piild/spe errc/iror: pc/chromium/s10rc/chrom9: e/comfile z:/tmon/ipc_channel_winas.cc, line k_156872341 +8986/build/[Child 8312, sCrc/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Parent 8392, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 4328, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ip[Parent 3308, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 2524, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 2524, Chrome_Ch[Child 5520, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 5520, Chrome_ChildThread] WARNING: pipe error: 109[Parent 3308, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 9292, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 9291570473487947 Marionette INFO Stopped listening on port 57490 +1570473544212 mozrunner::runner INFO Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "-marionette" "-foreground" "-no-remote" "-profile" "C:\\Users\\Admin\\AppData\\Local\\Temp\\rust_mozprofileQJP9IA" +1570473546067 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: mozillaAddons +1570473546068 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: telemetry +1570473546069 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: resource://pdf.js/ +1570473546070 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570473552282 Marionette INFO Listening on port 57645 +1570473552497 Marionette WARN TLS certificate errors will be ignored for this session +[Parent 8064, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +JavaScript warning: https://static.xx.fbcdn.net/rsrc.php/v3/y8/r/2VB0VDfSp1N.js?_nc_x=VgXmsmo3SSv, line 4: Error: WebGL warning: getContext: Disallowing antialiased backbuffers due to blacklisting. +JavaScript warning: https://static.xx.fbcdn.net/rsrc.php/v3iGIk4/yV/l/ru_RU/07lSJho0MKB.js?_nc_x=VgXmsmo3SSv, line 18: Error: WebGL warning: getContext: Disallowing antialiased backbuffers due to blacklisting. +JavaScript warning: https://static.xx.fbcdn.net/rsrc.php/v3iGIk4/yV/l/ru_RU/07lSJho0MKB.js?_nc_x=VgXmsmo3SSv, line 18: Error: WebGL warning: getContext: Disallowing antialiased backbuffers due to blacklisting. +[Parent 8064, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570473564779 Marionette INFO Listening on port 57681 +1570473565259 Marionette WARN TLS certificate errors will be ignored for this session +1570473938918 mozrunner::runner INFO Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "-marionette" "-foreground" "-no-remote" "-profile" "C:\\Users\\Admin\\AppData\\Local\\Temp\\rust_mozprofile1h0Hoh" +1570473940794 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: mozillaAddons +1570473940795 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: telemetry +1570473940796 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: resource://pdf.js/ +1570473940796 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570473947386 Marionette INFO Listening on port 57767 +1570473947687 Marionette WARN TLS certificate errors will be ignored for this session +JavaScript warning: https://static.xx.fbcdn.net/rsrc.php/v3/y8/r/2VB0VDfSp1N.js?_nc_x=VgXmsmo3SSv, line 4: Error: WebGL warning: getContext: Disallowing antialiased backbuffers due to blacklisting. +JavaScript warning: https://static.xx.fbcdn.net/rsrc.php/v3iGIk4/yD/l/ru_RU/6_KXv7olM2l.js?_nc_x=VgXmsmo3SSv, line 18: Error: WebGL warning: getContext: Disallowing antialiased backbuffers due to blacklisting. +[Parent 8436, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +IPDL protocol error: Handler returned error code! + +###!!! [Parent][DispatchAsyncMessage] Error: PLayerTransaction::Msg_ReleaseLayer Processing error: message was deserialized, but the handler returned false (indicating failure) + +IPDL protocol error: Handler returned error code! + +###!!! [Parent][DispatchAsyncMessage] Error: PLayerTransaction::Msg_ReleaseLayer Processing error: message was deserialized, but the handler returned false (indicating failure) + +[Parent 8436, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 5184, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 5184, Chrom[Child 5104, Chrome_ChildTh[Parent 8436, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 3744, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 31570474196702 Marionette INFO Stopped listening on port 57767 +ile z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 10792, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 10791570474196625 Marionette INFO Stopped listening on port 57808 +1570474214477 mozrunner::runner INFO Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "-marionette" "-foreground" "-no-remote" "-profile" "C:\\Users\\Admin\\AppData\\Local\\Temp\\rust_mozprofiledW26r8" +1570474216293 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: mozillaAddons +1570474216294 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: telemetry +1570474216295 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: resource://pdf.js/ +1570474216295 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570474222577 Marionette INFO Listening on port 57894 +1570474222729 Marionette WARN TLS certificate errors will be ignored for this session +JavaScript error: https://www.google.com/recaptcha/api2/anchor?ar=1&k=6Lc9qjcUAAAAADTnJq5kJMjN9aD1lxpRLMnCS2TR&co=aHR0cHM6Ly9mYnNieC5jb206NDQz&hl=ru&v=Zy-zVXWdnDW6AUZkKlojAKGe&size=normal&cb=inj1xh1efnh5, line 182: ReferenceError: recaptcha is not defined +JavaScript warning: https://static.xx.fbcdn.net/rsrc.php/v3/y8/r/2VB0VDfSp1N.js?_nc_x=VgXmsmo3SSv, line 4: Error: WebGL warning: getContext: Disallowing antialiased backbuffers due to blacklisting. +JavaScript warning: https://static.xx.fbcdn.net/rsrc.php/v3iGIk4/yD/l/ru_RU/6_KXv7olM2l.js?_nc_x=VgXmsmo3SSv, line 18: Error: WebGL warning: getContext: Disallowing antialiased backbuffers due to blacklisting. +[Parent 556, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +creenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570474234506 Marionette INFO Listening on port 57930 +1570474234895 Marionette WARN TLS certificate errors will be ignored for this session +1570474568167 mozrunner::runner INFO Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "-marionette" "-foreground" "-no-remote" "-profile" "C:\\Users\\Admin\\AppData\\Local\\Temp\\rust_mozprofilem3K4ZL" +1570474570066 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: mozillaAddons +1570474570066 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: telemetry +1570474570067 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: resource://pdf.js/ +1570474570068 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570474576364 Marionette INFO Listening on port 58009 +1570474576427 Marionette WARN TLS certificate errors will be ignored for this session +JavaScript error: https://www.google.com/recaptcha/api2/bframe?hl=ru&v=Zy-zVXWdnDW6AUZkKlojAKGe&k=6Lc9qjcUAAAAADTnJq5kJMjN9aD1lxpRLMnCS2TR&cb=1wcawx1phagc, line 184: ReferenceError: recaptcha is not defined +[Parent 8984, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Parent 8984, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Parent 8984, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 9080, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 9080, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/s[Parent 8984, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 5276, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568[Parent 4168, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Parent 4168, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 4604, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 4604,[Child 11080, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 11080, Chrome_ChildT[Parent 4168, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 10744, Chrome_ChildThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +[Child 10744, C1570474639701 Marionette INFO Stopped listening on port 58045 +1570474654800 mozrunner::runner INFO Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "-marionette" "-foreground" "-no-remote" "-profile" "C:\\Users\\Admin\\AppData\\Local\\Temp\\rust_mozprofilePZKx3x" +1570474656649 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: mozillaAddons +1570474656650 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: telemetry +1570474656651 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: resource://pdf.js/ +1570474656652 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570474662691 Marionette INFO Listening on port 58119 +1570474663057 Marionette WARN TLS certificate errors will be ignored for this session +[Parent 8336, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +JavaScript warning: https://static.xx.fbcdn.net/rsrc.php/v3/y8/r/2VB0VDfSp1N.js?_nc_x=VgXmsmo3SSv, line 4: Error: WebGL warning: getContext: Disallowing antialiased backbuffers due to blacklisting. +JavaScript warning: https://static.xx.fbcdn.net/rsrc.php/v3iGIk4/yV/l/ru_RU/07lSJho0MKB.js?_nc_x=VgXmsmo3SSv, line 18: Error: WebGL warning: getContext: Disallowing antialiased backbuffers due to blacklisting. +JavaScript warning: https://static.xx.fbcdn.net/rsrc.php/v3iGIk4/yV/l/ru_RU/07lSJho0MKB.js?_nc_x=VgXmsmo3SSv, line 18: Error: WebGL warning: getContext: Disallowing antialiased backbuffers due to blacklisting. +[Parent 8336, Gecko_IOThread] WARNING: pipe error: 109: file z:/task_1568728986/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 341 +resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570474675105 Marionette INFO Listening on port 58155 +1570474675269 Marionette WARN TLS certificate errors will be ignored for this session +1570474960741 mozrunner::runner INFO Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "-marionette" "-foreground" "-no-remote" "-profile" "C:\\Users\\Admin\\AppData\\Local\\Temp\\rust_mozprofilez41ya3" +1570474962630 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: mozillaAddons +1570474962631 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: telemetry +1570474962632 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: resource://pdf.js/ +1570474962633 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570474968897 Marionette INFO Listening on port 58237 +1570474969005 Marionette WARN TLS certificate errors will be ignored for this session +JavaScript warning: https://www.facebook.com/rsrc.php/v3iGIk4/yD/l/ru_RU/6_KXv7olM2l.js?_nc_x=VgXmsmo3SSv, line 18: Error: WebGL warning: getContext: Disallowing antialiased backbuffers due to blacklisting. +fileyKXRtm" +1570474974266 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: mozillaAddons +1570474974267 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: telemetry +1570474974268 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: resource://pdf.js/ +1570474974268 addons.webextension.screenshots@mozilla.org WARN Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader* +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1570474980907 Marionette INFO Listening on port 58273 +1570474981252 Marionette WARN TLS certificate errors will be ignored for this session diff --git a/jobparser/__init__.py b/jobparser/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/jobparser/__pycache__/__init__.cpython-37.pyc b/jobparser/__pycache__/__init__.cpython-37.pyc new file mode 100644 index 0000000..4230ab4 Binary files /dev/null and b/jobparser/__pycache__/__init__.cpython-37.pyc differ diff --git a/jobparser/__pycache__/items.cpython-37.pyc b/jobparser/__pycache__/items.cpython-37.pyc new file mode 100644 index 0000000..fec127f Binary files /dev/null and b/jobparser/__pycache__/items.cpython-37.pyc differ diff --git a/jobparser/__pycache__/settings.cpython-37.pyc b/jobparser/__pycache__/settings.cpython-37.pyc new file mode 100644 index 0000000..eacf955 Binary files /dev/null and b/jobparser/__pycache__/settings.cpython-37.pyc differ diff --git a/jobparser/items.py b/jobparser/items.py new file mode 100644 index 0000000..9d51e10 --- /dev/null +++ b/jobparser/items.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- + +# Define here the models for your scraped items +# +# See documentation in: +# https://docs.scrapy.org/en/latest/topics/items.html + +import scrapy +from scrapy.loader.processors import MapCompose, TakeFirst + + +def cleaner_photo(values): + if values[:2] == '//': + return f'http:{values}' + return values + +def cleaner_data(values): + values = values.replace(' ', '') + if '\xa0м²' in values: + return float(values.replace('\xa0м²', '')) + if '6' in values: + return int(values.replace('6-комнатные', '6')) + if '5' in values: + return int(values.replace('5-комнатные', '5')) + if '4' in values: + return int(values.replace('4-комнатные', '4')) + if '3' in values: + return int(values.replace('3-комнатные', '3')) + if '2' in values: + return int(values.replace('2-комнатные', '2')) + if '1' in values: + return int(values.replace('1-комнатные', '1')) + if 'студии' in values: + return float(values.replace('студии', '0.5')) + + + +class JobparserItem(scrapy.Item): + # define the fields for your item here like: + _id = scrapy.Field() + name = scrapy.Field() + salary = scrapy.Field() + company = scrapy.Field() + + +class InsJPItem(scrapy.Item): + _id = scrapy.Field() + user_name = scrapy.Field() + post_shortcode = scrapy.Field() + post_comments = scrapy.Field() + post_likes = scrapy.Field() + + +class AvitoRealEstate(scrapy.Item): + _id = scrapy.Field() + photos = scrapy.Field(input_processor=MapCompose(cleaner_photo)) + title = scrapy.Field(output_processor=TakeFirst()) + floor = scrapy.Field(output_processor=TakeFirst(), input_processor=MapCompose(cleaner_data)) + house_floors = scrapy.Field(output_processor=TakeFirst(), input_processor=MapCompose(cleaner_data)) + house_type = scrapy.Field(output_processor=TakeFirst(), input_processor=MapCompose(cleaner_data)) + rooms = scrapy.Field(output_processor=TakeFirst(), input_processor=MapCompose(cleaner_data)) + total_s = scrapy.Field(output_processor=TakeFirst(), input_processor=MapCompose(cleaner_data)) + living_s = scrapy.Field(output_processor=TakeFirst(), input_processor=MapCompose(cleaner_data)) + kitchen_s = scrapy.Field(output_processor=TakeFirst(), input_processor=MapCompose(cleaner_data)) diff --git a/jobparser/middlewares.py b/jobparser/middlewares.py new file mode 100644 index 0000000..f6cc61d --- /dev/null +++ b/jobparser/middlewares.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- + +# Define here the models for your spider middleware +# +# See documentation in: +# https://docs.scrapy.org/en/latest/topics/spider-middleware.html + +from scrapy import signals + + +class JobparserSpiderMiddleware(object): + # Not all methods need to be defined. If a method is not defined, + # scrapy acts as if the spider middleware does not modify the + # passed objects. + + @classmethod + def from_crawler(cls, crawler): + # This method is used by Scrapy to create your spiders. + s = cls() + crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) + return s + + def process_spider_input(self, response, spider): + # Called for each response that goes through the spider + # middleware and into the spider. + + # Should return None or raise an exception. + return None + + def process_spider_output(self, response, result, spider): + # Called with the results returned from the Spider, after + # it has processed the response. + + # Must return an iterable of Request, dict or Item objects. + for i in result: + yield i + + def process_spider_exception(self, response, exception, spider): + # Called when a spider or process_spider_input() method + # (from other spider middleware) raises an exception. + + # Should return either None or an iterable of Request, dict + # or Item objects. + pass + + def process_start_requests(self, start_requests, spider): + # Called with the start requests of the spider, and works + # similarly to the process_spider_output() method, except + # that it doesn’t have a response associated. + + # Must return only requests (not items). + for r in start_requests: + yield r + + def spider_opened(self, spider): + spider.logger.info('Spider opened: %s' % spider.name) + + +class JobparserDownloaderMiddleware(object): + # Not all methods need to be defined. If a method is not defined, + # scrapy acts as if the downloader middleware does not modify the + # passed objects. + + @classmethod + def from_crawler(cls, crawler): + # This method is used by Scrapy to create your spiders. + s = cls() + crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) + return s + + def process_request(self, request, spider): + # Called for each request that goes through the downloader + # middleware. + + # Must either: + # - return None: continue processing this request + # - or return a Response object + # - or return a Request object + # - or raise IgnoreRequest: process_exception() methods of + # installed downloader middleware will be called + return None + + def process_response(self, request, response, spider): + # Called with the response returned from the downloader. + + # Must either; + # - return a Response object + # - return a Request object + # - or raise IgnoreRequest + return response + + def process_exception(self, request, exception, spider): + # Called when a download handler or a process_request() + # (from other downloader middleware) raises an exception. + + # Must either: + # - return None: continue processing this exception + # - return a Response object: stops process_exception() chain + # - return a Request object: stops process_exception() chain + pass + + def spider_opened(self, spider): + spider.logger.info('Spider opened: %s' % spider.name) diff --git a/jobparser/pipelines.py b/jobparser/pipelines.py new file mode 100644 index 0000000..7b1bbe2 --- /dev/null +++ b/jobparser/pipelines.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- + +# Define your item pipelines here +# +# Don't forget to add your pipeline to the ITEM_PIPELINES setting +# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html +from pymongo import MongoClient +from database.base import VacancyDB +from database.models import Vacancy +import scrapy +from scrapy.pipelines.images import ImagesPipeline + + +class JobparserPipeline(object): + def __init__(self): + client = MongoClient('localhost', 27017) + self.mongo_base = client.vacancy + self.sql_db = VacancyDB('sqlite:///vacancy.sqlite') + + def process_item(self, item, spider): + collection = self.mongo_base[spider.name] + collection.insert_one(item) + # db_item = Vacancy(name=item.get("name"), spider=spider.name, salary=item.get("salary"), company=item.get('company')) + # self.sql_db.add_salery(db_item) + return item + + +class AvitoPhotosPipelines(ImagesPipeline): + def get_media_requests(self, item, info): + if item['photos']: + for img in item['photos']: + try: + yield scrapy.Request(img) + except TypeError: + pass + + def item_completed(self, results, item, info): + if results: + item['photos'] = [itm[1] for itm in results if itm[0]] + return item diff --git a/jobparser/settings.py b/jobparser/settings.py new file mode 100644 index 0000000..48fd4a9 --- /dev/null +++ b/jobparser/settings.py @@ -0,0 +1,95 @@ +# -*- coding: utf-8 -*- + +# Scrapy settings for jobparser project +# +# For simplicity, this file contains only settings considered important or +# commonly used. You can find more settings consulting the documentation: +# +# https://docs.scrapy.org/en/latest/topics/settings.html +# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html +# https://docs.scrapy.org/en/latest/topics/spider-middleware.html + +BOT_NAME = 'jobparser' + +SPIDER_MODULES = ['jobparser.spiders'] +NEWSPIDER_MODULE = 'jobparser.spiders' + +LOG_ENABELED = True +LOG_LEVEL = 'DEBUG' +# LOG_FILE = 'my_first_pider.txt' + +# Crawl responsibly by identifying yourself (and your website) on the user-agent +USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:69.0) Gecko/20100101 Firefox/69.0' + +# Obey robots.txt rules +ROBOTSTXT_OBEY = False + +# Configure maximum concurrent requests performed by Scrapy (default: 16) +CONCURRENT_REQUESTS = 1 + +# Configure a delay for requests for the same website (default: 0) +# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay +# See also autothrottle settings and docs +DOWNLOAD_DELAY = 3 +# The download delay setting will honor only one of: +#CONCURRENT_REQUESTS_PER_DOMAIN = 16 +#CONCURRENT_REQUESTS_PER_IP = 16 + +# Disable cookies (enabled by default) +COOKIES_ENABLED = False + +# Disable Telnet Console (enabled by default) +#TELNETCONSOLE_ENABLED = False + +# Override the default request headers: +#DEFAULT_REQUEST_HEADERS = { +# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', +# 'Accept-Language': 'en', +#} + +# Enable or disable spider middlewares +# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html +#SPIDER_MIDDLEWARES = { +# 'jobparser.middlewares.JobparserSpiderMiddleware': 543, +#} + +# Enable or disable downloader middlewares +# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html +#DOWNLOADER_MIDDLEWARES = { +# 'jobparser.middlewares.JobparserDownloaderMiddleware': 543, +#} + +# Enable or disable extensions +# See https://docs.scrapy.org/en/latest/topics/extensions.html +#EXTENSIONS = { +# 'scrapy.extensions.telnet.TelnetConsole': None, +#} + +# Configure item pipelines +# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html +ITEM_PIPELINES = { + 'jobparser.pipelines.AvitoPhotosPipelines': 100, + 'jobparser.pipelines.JobparserPipeline': 300, +} + +IMAGES_STORE = 'images' +# Enable and configure the AutoThrottle extension (disabled by default) +# See https://docs.scrapy.org/en/latest/topics/autothrottle.html +#AUTOTHROTTLE_ENABLED = True +# The initial download delay +#AUTOTHROTTLE_START_DELAY = 5 +# The maximum download delay to be set in case of high latencies +#AUTOTHROTTLE_MAX_DELAY = 60 +# The average number of requests Scrapy should be sending in parallel to +# each remote server +#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 +# Enable showing throttling stats for every response received: +#AUTOTHROTTLE_DEBUG = False + +# Enable and configure HTTP caching (disabled by default) +# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings +#HTTPCACHE_ENABLED = True +#HTTPCACHE_EXPIRATION_SECS = 0 +#HTTPCACHE_DIR = 'httpcache' +#HTTPCACHE_IGNORE_HTTP_CODES = [] +#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' diff --git a/jobparser/spiders/__init__.py b/jobparser/spiders/__init__.py new file mode 100644 index 0000000..ebd689a --- /dev/null +++ b/jobparser/spiders/__init__.py @@ -0,0 +1,4 @@ +# This package will contain the spiders of your Scrapy project +# +# Please refer to the documentation for information on how to create and manage +# your spiders. diff --git a/jobparser/spiders/__pycache__/__init__.cpython-37.pyc b/jobparser/spiders/__pycache__/__init__.cpython-37.pyc new file mode 100644 index 0000000..77ca937 Binary files /dev/null and b/jobparser/spiders/__pycache__/__init__.cpython-37.pyc differ diff --git a/jobparser/spiders/__pycache__/avito.cpython-37.pyc b/jobparser/spiders/__pycache__/avito.cpython-37.pyc new file mode 100644 index 0000000..f848c5d Binary files /dev/null and b/jobparser/spiders/__pycache__/avito.cpython-37.pyc differ diff --git a/jobparser/spiders/__pycache__/avt.cpython-37.pyc b/jobparser/spiders/__pycache__/avt.cpython-37.pyc new file mode 100644 index 0000000..cdea8d6 Binary files /dev/null and b/jobparser/spiders/__pycache__/avt.cpython-37.pyc differ diff --git a/jobparser/spiders/__pycache__/hhru.cpython-37.pyc b/jobparser/spiders/__pycache__/hhru.cpython-37.pyc new file mode 100644 index 0000000..266816f Binary files /dev/null and b/jobparser/spiders/__pycache__/hhru.cpython-37.pyc differ diff --git a/jobparser/spiders/__pycache__/instagram.cpython-37.pyc b/jobparser/spiders/__pycache__/instagram.cpython-37.pyc new file mode 100644 index 0000000..fe06ffa Binary files /dev/null and b/jobparser/spiders/__pycache__/instagram.cpython-37.pyc differ diff --git a/jobparser/spiders/__pycache__/sjru.cpython-37.pyc b/jobparser/spiders/__pycache__/sjru.cpython-37.pyc new file mode 100644 index 0000000..3556a0e Binary files /dev/null and b/jobparser/spiders/__pycache__/sjru.cpython-37.pyc differ diff --git a/jobparser/spiders/__pycache__/superjob.cpython-37.pyc b/jobparser/spiders/__pycache__/superjob.cpython-37.pyc new file mode 100644 index 0000000..2b48ff5 Binary files /dev/null and b/jobparser/spiders/__pycache__/superjob.cpython-37.pyc differ diff --git a/jobparser/spiders/__pycache__/zillow.cpython-37.pyc b/jobparser/spiders/__pycache__/zillow.cpython-37.pyc new file mode 100644 index 0000000..15ecfcd Binary files /dev/null and b/jobparser/spiders/__pycache__/zillow.cpython-37.pyc differ diff --git a/jobparser/spiders/avito.py b/jobparser/spiders/avito.py new file mode 100644 index 0000000..d1da8e1 --- /dev/null +++ b/jobparser/spiders/avito.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +import scrapy +from scrapy.http import HtmlResponse +from jobparser.items import JobparserItem + +class AvitoSpider(scrapy.Spider): + name = 'avito' + allowed_domains = ['avito.ru'] + start_urls = ['https://www.avito.ru/rossiya/rabota?q=Python'] + + def parse(self, response: HtmlResponse): + vacancy_urls = response.xpath('//a[@class="item-description-title-link"]/@href').extract() + next_page = response.xpath('//a[contains(@class, "js-pagination-next")]/@href').extract_first() + yield response.follow(next_page, callback=self.parse) + for vac in vacancy_urls: + yield response.follow(vac, callback=self.parse_vacancy) + + def parse_vacancy(self, response: HtmlResponse): + _tmp_cur = {'₽': 'RUB', '$': 'USD'} + # _tmp_values = response.xpath("//div[@class='_3MVeX']/span[contains(@class, '_3mfro')]/span/text()").extract() + price = response.xpath("//span[@class='price-value-string js-price-value-string']/span[contains(@class,'js-item-price')]/text()").extract_first() + + if price: + price = price.replace(' ', '') + currency = response.xpath( + "//span[@class='price-value-string js-price-value-string']/span[contains(@class,'price-value-prices-list-item-currency_sign')]/span/text()").extract_first() + name = response.xpath("//span[@class='title-info-title-text']/text()").extract_first() + # name = response.xpath("//div[@class='_3MVeX']/h1/text()").extract_first() + # v_tmp = [int(itm.replace('\xa0', '')) for itm in _tmp_values[:-1] if itm.replace('\xa0', '').isdigit()] + salary = {'currency': currency if currency else None, + 'min_value': price if price else None, + 'max_value': None + } + # + # if salary['max_value']: + print(1) + # + yield JobparserItem(name=name, salary=salary) \ No newline at end of file diff --git a/jobparser/spiders/avt.py b/jobparser/spiders/avt.py new file mode 100644 index 0000000..c0a79e9 --- /dev/null +++ b/jobparser/spiders/avt.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- +import scrapy +from scrapy.http import HtmlResponse +from scrapy.loader import ItemLoader +from jobparser.items import AvitoRealEstate + +class AvtSpider(scrapy.Spider): + name = 'avito_ads' + allowed_domains = ['avito.ru'] + start_urls = ['https://avito.ru/rossiya/kvartiry'] + + def parse(self, response): + ads_links = response.xpath('//a[@class="item-description-title-link"]/@href').extract() + for link in ads_links: + yield response.follow(link, callback=self.parse_ads) + pass + + def parse_ads(self, response: HtmlResponse): + loader = ItemLoader(item=AvitoRealEstate(), response=response) + loader.add_xpath('photos', + '//div[contains(@class, "gallery-img-wrapper")]//div[contains(@class, "gallery-img-frame")]/@data-url') + loader.add_css('title', 'h1.title-info-title span.title-info-title-text::text') + par_names = response.css('li.item-params-list-item span.item-params-label::text').extract() + for i in range(len(par_names)): + par_names[i] = par_names[i].replace(' ', '') + + par_data = response.css('li.item-params-list-item::text').extract() + my_dict = {'Этаж:': 'floor', + 'Этажейвдоме:': 'house_floors', + 'Типдома:': 'house_type', + 'Количествокомнат:': 'rooms', + 'Общаяплощадь:': 'total_s', + 'Жилаяплощадь:': 'living_s', + 'Площадькухни:': 'kitchen_s', + 'Отделка:': 'otdelka' + } + + result_dict = {} + for i in range(len(par_names)): + result_dict[my_dict[par_names[i]]] = par_data[i * 2 + 1] + + for keys in result_dict: + loader.add_value(keys, result_dict[keys]) + print(1) + yield loader.load_item() + + diff --git a/jobparser/spiders/fb.py b/jobparser/spiders/fb.py new file mode 100644 index 0000000..f11b195 --- /dev/null +++ b/jobparser/spiders/fb.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +import scrapy +from scrapy.http import HtmlResponse +from selenium import webdriver +from selenium.webdriver.common.keys import Keys +import time +import random + +class FbSpider(scrapy.Spider): + fb_login = '' + fb_passwd = '' + name = 'fb' + allowed_domains = ['facebook.com'] + start_urls = ['https://www.facebook.com/'] + options = webdriver.FirefoxOptions() + options.set_preference("permissions.default.desktop-notification", 0) + webdriver = webdriver.Firefox(firefox_options=options) + initial_user = 'romanchuk.s.a' + sleep_min = 1 + sleep_max = 5 + + def parse(self, response: HtmlResponse): + self.webdriver.get('https://www.facebook.com/') + time.sleep(random.randint(self.sleep_min, self.sleep_max)) + self.webdriver.find_element_by_xpath('//input[@type="email"]').send_keys(self.fb_login) + self.webdriver.find_element_by_xpath('//input[@type="password"]').send_keys(self.fb_passwd) + self.webdriver.find_element_by_xpath('//input[@type="password"]').send_keys(Keys.RETURN) + time.sleep(random.randint(self.sleep_min, self.sleep_max)) + self.webdriver.get('https://www.facebook.com/romanchuk.s.a') + time.sleep(random.randint(self.sleep_min, self.sleep_max)) + self.webdriver.find_element_by_xpath('//body').send_keys(Keys.PAGE_DOWN) + self.webdriver.find_element_by_xpath('//body').send_keys(Keys.PAGE_DOWN) + time.sleep(random.randint(self.sleep_min, self.sleep_max)) + friends_url = self.webdriver.find_element_by_css_selector('#profile_timeline_tiles_unit_pagelets_friends a') + self.webdriver.get(friends_url.get_attribute('href')) + time.sleep(random.randint(self.sleep_min, self.sleep_max)) + counter_old = 0 + counter_new = 1 + while counter_old < counter_new: + self.webdriver.find_element_by_xpath('//body').send_keys(Keys.PAGE_DOWN) + self.webdriver.find_element_by_xpath('//body').send_keys(Keys.PAGE_DOWN) + self.webdriver.find_element_by_xpath('//body').send_keys(Keys.PAGE_DOWN) + self.webdriver.find_element_by_xpath('//body').send_keys(Keys.PAGE_DOWN) + time.sleep(random.randint(self.sleep_min, self.sleep_max)) + friends_profiles = self.webdriver.find_elements_by_xpath('//li/div[@class="clearfix _5qo4"]/a') + counter_old = counter_new + counter_new = len(friends_profiles) + time.sleep(random.randint(self.sleep_min, self.sleep_max)) + + for i in range(len(friends_profiles)): + yield self.parse_friend(friends_profiles[i].get_attribute('href')) + + def parse_friend(self, link): + self.webdriver.get(link) + time.sleep(random.randint(self.sleep_min, self.sleep_max)) + self.webdriver.find_element_by_xpath('//body').send_keys(Keys.PAGE_DOWN) + self.webdriver.find_element_by_xpath('//body').send_keys(Keys.PAGE_DOWN) + time.sleep(random.randint(self.sleep_min, self.sleep_max)) + self.webdriver.find_element_by_xpath('//body').send_keys(Keys.PAGE_DOWN) + self.webdriver.find_element_by_xpath('//body').send_keys(Keys.PAGE_DOWN) + time.sleep(random.randint(self.sleep_min, self.sleep_max)) + friends_url = self.webdriver.find_element_by_css_selector('#profile_timeline_tiles_unit_pagelets_friends a') + print(2) + pass +# webdriver.find_elements_by_xpath( + # '//ul[@class="media-stream"]/li/picture/source[@type="image/jpeg"]')) \ No newline at end of file diff --git a/jobparser/spiders/hhru.py b/jobparser/spiders/hhru.py new file mode 100644 index 0000000..bd2485e --- /dev/null +++ b/jobparser/spiders/hhru.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +import scrapy +from scrapy.http import HtmlResponse +from jobparser.items import JobparserItem + +class HhruSpider(scrapy.Spider): + name = 'hhru' + allowed_domains = ['hh.ru'] + start_urls = ['https://hh.ru/search/vacancy?area=113&st=searchVacancy&text=python'] + + def parse(self, response:HtmlResponse): + next_page = response.css("a.HH-Pager-Controls-Next::attr(href)").extract_first() + yield response.follow(next_page, callback=self.parse) + + vacancy = response.css('div.vacancy-serp div.vacancy-serp-item div.vacancy-serp-item__row_header a.bloko-link::attr(href)').extract() + for link in vacancy: + yield response.follow(link, callback=self.vacancy_parse) + pass + + def vacancy_parse(self, response: HtmlResponse): + name = response.css("div.vacancy-title h1.header::text").extract_first() + salary = response.css("div.vacancy-title p.vacancy-salary::text").extract_first() + company = ''.join(response.css('a.vacancy-company-name span::text').extract()) + yield JobparserItem(name=name, salary=salary, company=company) + diff --git a/jobparser/spiders/instagram.py b/jobparser/spiders/instagram.py new file mode 100644 index 0000000..6b9a979 --- /dev/null +++ b/jobparser/spiders/instagram.py @@ -0,0 +1,129 @@ +# -*- coding: utf-8 -*- +import re +import scrapy +from scrapy.http import HtmlResponse +import json +from urllib.parse import urlencode, urljoin +from copy import deepcopy +from jobparser.items import InsJPItem + +class InstagramSpider(scrapy.Spider): + name = 'instagram' + graphql_url = 'https://www.instagram.com/graphql/query/?' + allowed_domains = ['instagram.com'] + start_urls = ['https://instagram.com/'] + variables_base = {'fetch_mutual': 'false', "include_reel": 'true', "first": 100} + post_variables_base = {'child_comment_count': 3, 'fetch_comment_count': 40, 'parent_comment_count': 24, 'has_threaded_comments': 'true'} + followers = {} + posts = {} + post_comments = {} + post_commentors = [] + def __init__(self, user_links, login, pswrd, *args, **kwargs): + self.user_links = user_links + self.login = login + self.pswrd = pswrd + # self.query_hash = 'c76146de99bb02f6415203be841dd25a' + self.query_hash = 0 + super().__init__(*args, *kwargs) + + def parse(self, response: HtmlResponse): + csrf_token = self.fetch_csrf_token(response.text) + yield scrapy.FormRequest( + 'https://www.instagram.com/accounts/login/ajax/', + method='POST', + callback=self.parse_users, + formdata={'username': self.login, 'password': self.pswrd}, + headers={'X-CSRFToken': csrf_token} + ) + + def parse_users(self, response: HtmlResponse): + j_body = json.loads(response.body) + if j_body.get('authenticated'): + for user in self.user_links: + yield response.follow(urljoin(self.start_urls[0], user), + callback=self.parse_user, + cb_kwargs={'user': user}) + + def parse_user(self, response: HtmlResponse, user): + user_id = self.fetch_user_id(response.text, user) + user_vars = deepcopy(self.variables_base) + user_vars.update({'id': user_id}) + self.query_hash = '58b6785bea111c67129decbe6a448951' + yield response.follow(self.make_graphql_url(user_vars), + callback=self.parse_posts, + cb_kwargs={'user_vars': user_vars, 'user': user}) + # yield response.follow(self.make_graphql_url(user_vars), + # callback=self.parse_followers, + # cb_kwargs={'user_vars': user_vars, 'user': user}) + + def parse_posts(self, response: HtmlResponse, user_vars, user): + data = json.loads(response.body) + self.posts[user] = {'posts': data.get('data').get('user').get('edge_owner_to_timeline_media').get('edges')} + for i in range(10): + shortcode = self.posts.get(user).get('posts')[i].get('node').get('shortcode') + user_vars = deepcopy(self.post_variables_base) + user_vars.update({'shortcode': shortcode}) + self.query_hash = '865589822932d1b43dfe312121dd353a' + yield response.follow(self.make_graphql_url(user_vars), + callback=self.parse_post_commentors, + cb_kwargs={'user_vars': user_vars, 'user': user}) + + def parse_post_commentors(self, response: HtmlResponse, user_vars, user): + data = json.loads(response.body) + self.post_comments[user] = {'post_comments': data.get('data').get('shortcode_media').get('edge_media_to_parent_comment').get('edges')} + self.post_comments[user]['commentors'] = [] + if len(self.post_comments[user].get("post_comments"))>0: + if len(self.post_comments[user].get("post_comments"))>9: + limit = 9 + else: + limit = len(self.post_comments[user].get("post_comments")) + for i in range(limit): + self.post_comments[user]['commentors'].append(self.post_comments[user].get("post_comments")[i].get('node').get('owner').get('username')) + print(self.post_comments[user].get("post_comments")[i].get('node').get('owner').get('username')) + + yield InsJPItem(user_name=user, post_shortcode=user_vars['shortcode'], post_comments = self.post_comments[user]['commentors'], post_likes=None) + + + + def parse_followers(self, response: HtmlResponse, user_vars, user): + data = json.loads(response.body) + if not self.followers.get(user): + self.followers[user] = {'followers': data.get('data').get('user').get('edge_followed_by').get('edges'), + 'count': data.get('data').get('user').get('edge_followed_by').get('count')} + else: + self.followers[user]['followers'].extend(data.get('data').get('user').get('edge_followed_by').get('edges')) + + if data.get('data').get('user').get('edge_followed_by').get('page_info').get('has_next_page'): + user_vars.update( + {'after': data.get('data').get('user').get('edge_followed_by').get('page_info').get('end_cursor')}) + next_page = self.make_graphql_url(user_vars) + yield response.follow(next_page, callback=self.parse_followers, + cb_kwargs={'user_vars': user_vars, 'user': user}) + + if self.followers.get(user) and self.followers.get(user).get('count') == len(self.followers.get(user).get('followers')): + pass + + + + + + + + def fetch_csrf_token(self, text: HtmlResponse): + matched = re.search('\"csrf_token\":\"\\w+\"', text).group() + return matched.split(':').pop().replace(r'"', '') + + def fetch_user_id(self, text, username): + matched = re.search( + '{\"id\":\"\\d+\",\"username\":\"%s\"}' % username, text + + ).group() + return json.loads(matched).get('id') + + def make_graphql_url(self, user_vars): + result = '{url}query_hash={hash}&{variables}'.format( + url=self.graphql_url, hash=self.query_hash, + variables=urlencode(user_vars) + ) + print(result) + return result diff --git a/jobparser/spiders/sjru.py b/jobparser/spiders/sjru.py new file mode 100644 index 0000000..b9a0994 --- /dev/null +++ b/jobparser/spiders/sjru.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +import scrapy +from scrapy.http import HtmlResponse +from jobparser.items import JobparserItem + +class SjruSpider(scrapy.Spider): + name = 'sjru' + allowed_domains = ['superjob.ru'] + start_urls = ['https://www.superjob.ru/vacancy/search/?keywords=Python&geo%5Bc%5D%5B0%5D=1'] + + def parse(self, response:HtmlResponse): + next_page = response.css("a.f-test-link-dalshe::attr(href)").extract_first() + + yield response.follow(next_page, callback=self.parse) + + vacancy = response.css("div.f-test-vacancy-item div._2g1F- a._1QIBo::attr(href)").extract() + for link in vacancy: + yield response.follow(link, callback=self.vacancy_parse) + pass + + def vacancy_parse(self, response: HtmlResponse): + name = response.css("div._3MVeX h1._3mfro::text").extract_first() + salary = ''.join(response.css("div._3MVeX span._2Wp8I::text").extract()) + company = response.css("h2.PlM3e::text").extract_first() + yield JobparserItem(name=name, salary=salary, company=company) + diff --git a/jobparser/spiders/superjob.py b/jobparser/spiders/superjob.py new file mode 100644 index 0000000..e309b37 --- /dev/null +++ b/jobparser/spiders/superjob.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +import scrapy +from scrapy.http import HtmlResponse +from jobparser.items import JobparserItem + +class SuperjobSpider(scrapy.Spider): + name = 'superjob' + allowed_domains = ['superjob.ru'] + start_urls = ['https://www.superjob.ru/vacancy/search/?keywords=python&geo%5Bc%5D%5B0%5D=1'] + + def parse(self, response: HtmlResponse): + vacancy_urls = response.xpath('//a[contains(@class, "icMQ_") and contains(@class, "_1QIBo")]/@href').extract() + next_page = response.xpath('//div[contains(@class, "L1p51")]/a[@rel="next"]/@href').extract_first() + yield response.follow(next_page, callback=self.parse) + for vac in vacancy_urls: + yield response.follow(vac, callback=self.parse_vacancy) + + def parse_vacancy(self, response: HtmlResponse): + _tmp_cur = {'₽': 'RUB', '$': 'USD'} + _tmp_values = response.xpath("//div[@class='_3MVeX']/span[contains(@class, '_3mfro')]/span/text()").extract() + name = response.xpath("//div[@class='_3MVeX']/h1/text()").extract_first() + v_tmp = [int(itm.replace('\xa0', '')) for itm in _tmp_values[:-1] if itm.replace('\xa0', '').isdigit()] + salary = {'currency': (lambda x: _tmp_cur[x] if x and x in _tmp_cur else None)( + _tmp_values[-1]) if _tmp_values else None, + 'min_value': v_tmp[0] if v_tmp else None, + 'max_value': v_tmp[1] if v_tmp and len(v_tmp) > 1 else None + } + + if salary['max_value']: + print(1) + + yield JobparserItem(name=name, salary=salary) \ No newline at end of file diff --git a/jobparser/spiders/zillow.py b/jobparser/spiders/zillow.py new file mode 100644 index 0000000..4aca6d4 --- /dev/null +++ b/jobparser/spiders/zillow.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +import scrapy +from scrapy.http import HtmlResponse +from jobparser.items import AvitoRealEstate +from selenium import webdriver +from selenium.webdriver.common.keys import Keys + + +class ZillowSpider(scrapy.Spider): + name = 'zillow' + allowed_domains = ['zillow.com'] + start_urls = ['https://www.zillow.com/fort-worth-tx/'] + webdriver = webdriver.Firefox() + + #def __init__(self): + #self.webdriver = webdriver.Firefox() + #super().__init__(self) + + def parse(self, response: HtmlResponse): + real_estate_list = response.css('div#grid-search-results ul.photo-cards li article a.list-card-link::attr(href)') + next = response.css(".zsg-pagination-next a::attr(href)").extract_first() + yield response.follow(next, callback=self.parse) + + for adv in real_estate_list.extract(): + yield self.parse_adv(adv) + + def parse_adv(self, link): + self.webdriver.get(link) + media = self.webdriver.find_element_by_css_selector('.ds-media-col') + photos_ul = self.webdriver.find_elements_by_css_selector('.ds-media-col ul.media-stream li') + photo_pic_img = self.webdriver.find_elements_by_xpath( + '//ul[@class="media-stream"]/li/picture/source[@type="image/jpeg"]') + while len(photos_ul) > (len(photo_pic_img)-5): + media.send_keys(Keys.PAGE_DOWN) + photo_pic_img = self.webdriver.find_elements_by_xpath( + '//ul[@class="media-stream"]/li/picture/source[@type="image/jpeg"]') + print(self.webdriver.title, self.webdriver.current_window_handle) + return None \ No newline at end of file diff --git a/lesson1.py b/lesson1.py new file mode 100644 index 0000000..a258cfd --- /dev/null +++ b/lesson1.py @@ -0,0 +1,18 @@ +import requests + +USER_AGENT = 'Mozilla/4.0 (compatible; MSIE 6.0; America Online Browser 1.1; Windows NT 5.1; FunWebProducts)' + +github_api_url = 'https://api.github.com/users/' + +user_name='Belfi-Gor' + +response = requests.get(f'{github_api_url}{user_name}/repos', headers={'User-Agent': USER_AGENT}) + +data = response.json() + +rep_list = {} + +print(data) + +for row in data: + print(f'{row["name"]} - {row["description"]}') \ No newline at end of file diff --git a/runner.py b/runner.py new file mode 100644 index 0000000..c7eddf1 --- /dev/null +++ b/runner.py @@ -0,0 +1,32 @@ +import os +from os.path import join, dirname +from dotenv import load_dotenv +from scrapy.crawler import CrawlerProcess +from scrapy.settings import Settings + +from jobparser import settings +#from jobparser.spiders.hhru import HhruSpider +#from jobparser.spiders.sjru import SjruSpider +#from jobparser.spiders.instagram import InstagramSpider +#from jobparser.spiders.superjob import SuperjobSpider +#from jobparser.spiders.avito import AvitoSpider +#from jobparser.spiders.avt import AvtSpider +#from jobparser.spiders.zillow import ZillowSpider +from jobparser.spiders.fb import FbSpider +do_env = join(dirname(__file__), '.env') +load_dotenv(do_env) + +INST_LOGIN = os.getenv('INST_LOGIN') +INST_PWD = os.getenv('INST_PASSWORD') + +if __name__ == '__main__': + crawler_setting = Settings() + crawler_setting.setmodule(settings) + process = CrawlerProcess(settings=crawler_setting) + # process.crawl(HhruSpider) + # process.crawl(SjruSpider) + # process.crawl(InstagramSpider, ['geekbrains.ru'], INST_LOGIN, INST_PWD) + # process.crawl(SuperjobSpider) + # process.crawl(ZillowSpider) + process.crawl(FbSpider) + process.start() diff --git a/scrapy.cfg b/scrapy.cfg new file mode 100644 index 0000000..0e36c64 --- /dev/null +++ b/scrapy.cfg @@ -0,0 +1,11 @@ +# Automatically created by: scrapy startproject +# +# For more information about the [deploy] section see: +# https://scrapyd.readthedocs.io/en/latest/deploy.html + +[settings] +default = jobparser.settings + +[deploy] +#url = http://localhost:6800/ +project = jobparser diff --git a/vacancy.sqlite b/vacancy.sqlite new file mode 100644 index 0000000..526f323 Binary files /dev/null and b/vacancy.sqlite differ