", "
", "p>") + for link in self.links: + if link == self.link: + self.links.remove(link) + except Exception as e: + logging.warning("Problems with tag parsing:\n" + str(e)) + + def __str__(self): + string = "Channel name: {0}\n" \ + "Title: {1}\n" \ + "Date: {2}\n" \ + "Link: {3}\n\n" \ + "Summary: {4}".format(self.channel_name, self.title, self.date, self.link, self.summary) + if len(self.images) > 0: + string = string + "\n\nImages in the article:" + for img in self.images: + string = string + "\n" + img + if len(self.links) > 0: + string = string + "\n\nLinks in the article:" + for link in self.links: + string = string + "\n" + link + return string diff --git a/App/Portal.py b/App/Portal.py new file mode 100644 index 0000000..6fc7cfd --- /dev/null +++ b/App/Portal.py @@ -0,0 +1,103 @@ +import logging +import feedparser +import json +from App.Errors import FatalError +from App.News import News +from App.ToHtml import ToHtml +from App.ToPDF import ToPDF +from termcolor import colored +from App.Colors import Colors + + +class Portal: + """The class is used to store and process information associated with one news portal""" + + def __init__(self, url, limit): + logging.info("Creating object Portal") + self.url = url + rss = self.get_rss() + try: + self.title = rss.feed.title + self.link = rss.feed.link + self.updated = None + self.news = [] + self.limit = limit + self.links = [] + self.update(rss.entries[::-1]) + except Exception as e: + raise FatalError("Problems with rss processing") + + def get_rss(self): + """Get rss file""" + logging.info("Getting rss file") + try: + return feedparser.parse(self.url) + except Exception as e: + raise FatalError("Problems getting rss file") + + def update(self, entries): + """The method is used to obtain articles""" + logging.info("Start processing article") + if self.limit is None or self.limit > len(entries): + limit = len(entries) + else: + limit = self.limit + try: + rss = self.get_rss() + if self.updated != rss.feed.updated: + self.updated = rss.feed.updated + for entry in entries[:limit]: + self.news.insert(0, News(entry, self.title)) + except FatalError: + raise + except Exception as e: + raise FatalError("Problems with article processing") + + def load_new_news(self, news): + if self.limit is None or self.limit > len(news): + self.news = news + else: + self.news = news[:self.limit] + + def print(self, json_flag): + """The method displays information about the portal and articles""" + try: + if json_flag: + logging.info("Saving to json") + json_news = [] + for news in self.news: + json_news.append({"Title": news.title, "Date": news.date, "Link": news.link, + "Summary": news.summary, "Images": news.images, "Links": news.links}) + main_dict = {"Title": self.title, "Url": self.url, "News": json_news} + + print(json.dumps(main_dict, ensure_ascii=False, indent=4)) + else: + logging.info("Saving to text") + print(colored("\n\nRSS-chanel", Colors["other"])) + for news in self.news: + print(colored("\n" + "*" * 20 + "New article" + "*" * 20 + "\n", Colors["article"])) + print(colored(news, Colors["text"])) + except Exception as e: + logging.error(str(e)) + raise FatalError("Problems with printing") + + def convert_to_html(self, html_path): + """Convert news to html""" + logging.info("Start converting news to html") + try: + to_html = ToHtml(self.news, html_path) + to_html.make_file() + except Exception as e: + print(colored("Error with converting to html", Colors["error"])) + logging.info(str(e)) + + def convert_to_pdf(self, pdf_path): + """Convert news to pdf""" + logging.info("Start converting news to pdf") + try: + to_html = ToHtml(self.news) + to_pdf = ToPDF(to_html.html, pdf_path) + to_pdf.make_file() + except Exception as e: + print(colored("Error with converting to pdf", Colors["error"])) + logging.info(str(e)) diff --git a/App/RSSListener.py b/App/RSSListener.py new file mode 100644 index 0000000..ad2f552 --- /dev/null +++ b/App/RSSListener.py @@ -0,0 +1,48 @@ +import logging +from App.Portal import Portal +from App.Errors import FatalError +from App.Saver import Saver +from termcolor import colored +from App.Colors import Colors + + +class RSSListener: + """Class listener""" + + def __init__(self, limit, json_flag, date, html_path, pdf_path): + logging.info("Creating object RSSListener") + self.limit = limit + self.date = date + self.portal = None + self.json_flag = json_flag + self.html_path = html_path + self.pdf_path = pdf_path + + def start(self, url): + """Class listener. Handles new rss links and saved news""" + logging.info("We begin to process the url") + try: + self.portal = Portal(url, self.limit) + saver = Saver() + saver.start_saving(self.portal.news) + if self.date is not None: + old_news = saver.load(self.date) + if old_news is not None: + self.portal.load_new_news(old_news) + self.printing() + else: + print(colored("Error: news haven't been founded", Colors["error"])) + + else: + self.printing() + if self.html_path is not None: + self.portal.convert_to_html(self.html_path) + if self.pdf_path is not None: + self.portal.convert_to_pdf(self.pdf_path) + except FatalError: + raise + except Exception as e: + raise FatalError("Something go wrong") + + def printing(self): + self.portal.print(self.json_flag) diff --git a/App/Saver.py b/App/Saver.py new file mode 100644 index 0000000..e2596e0 --- /dev/null +++ b/App/Saver.py @@ -0,0 +1,54 @@ +import pickle +import os +import logging + + +class Saver: + """The class is responsible for saving and unloading data.""" + def sort(self, news_list): + """The method sorts news from the link by date""" + logging.info("Sorting news in saver") + date_handler = {} + for news in news_list: + if news.parsed_date in date_handler: + date_handler[news.parsed_date].append(news) + else: + date_handler[news.parsed_date] = [news, ] + return date_handler + + def save(self, date_handler): + """Save data""" + logging.info("Saving data") + for date in date_handler: + if os.path.exists("./Cache/" + date): + with open("./Cache/" + date, 'rb') as f: + old_date = pickle.load(f) + delete_list = [] + for new_d in date_handler[date]: + for old_d in old_date: + if str(new_d) == str(old_d): + delete_list.append(new_d) + for new_d in delete_list: + date_handler[date].remove(new_d) + with open("./Cache/" + date, 'wb') as f: + pickle.dump(old_date + date_handler[date], f) + else: + with open("./Cache/" + date, 'wb') as f: + pickle.dump(date_handler[date], f) + + def start_saving(self, news_list): + try: + handler = self.sort(news_list) + self.save(handler) + except Exception as e: + logging.error("Saving error") + logging.error(str(e)) + + def load(self, date): + """Load data from files""" + logging.info("Loading data from files") + if not os.path.exists("./Cache/" + date): + return None + with open("./Cache/" + date, 'rb') as f: + old_news = pickle.load(f) + return old_news diff --git a/App/ToHtml.py b/App/ToHtml.py new file mode 100644 index 0000000..a9cff74 --- /dev/null +++ b/App/ToHtml.py @@ -0,0 +1,48 @@ +import logging +from termcolor import colored +from App.Colors import Colors + + +class ToHtml: + """Class responsible for converting data to html""" + def __init__(self, news, path="./news.html"): + self.news = news + self.path = path + self.html = self.make_html() + + def make_html(self): + """Create html""" + logging.info("Creating html") + html = """ + + +
+{entry.title}
" + html += f"Channel name: {entry.channel_name}
" + html += f"Date: {entry.date}
" + html += f"" + for img in entry.images: + html += f"{entry.summary}
" + if len(entry.links) > 0: + counter = 0 + html += "Links in the article:
" + for link in entry.links: + counter += 1 + html += f"" + html += "" + return html + + def make_file(self): + """Create html file""" + logging.info("Creating html file") + try: + with open(self.path, 'w') as f: + f.write(self.html) + except: + print(colored("Saving file error. Problems with path", Colors["error"])) diff --git a/App/ToPDF.py b/App/ToPDF.py new file mode 100644 index 0000000..c13ced1 --- /dev/null +++ b/App/ToPDF.py @@ -0,0 +1,26 @@ +import weasyprint +import logging +from termcolor import colored +from App.Colors import Colors + + +class ToPDF: + """Class responsible for converting data to pdf""" + def __init__(self, html, path="./news.pdf"): + self.html = html + self.path = path + self.pdf = self.make_pdf() + + def make_pdf(self): + """Create pdf""" + logging.info("Creating pdf") + return weasyprint.HTML(string=self.html).write_pdf() + + def make_file(self): + """Create pdf file""" + logging.info("Creating pdf file") + try: + with open(self.path, "wb") as f: + f.write(self.pdf) + except: + print(colored("Saving file error. Problems with path", Colors["error"])) diff --git a/App/__init__.py b/App/__init__.py new file mode 100644 index 0000000..a7fd423 --- /dev/null +++ b/App/__init__.py @@ -0,0 +1 @@ +__version__ = "5.0" diff --git a/App/rss_reader.py b/App/rss_reader.py new file mode 100755 index 0000000..f23481d --- /dev/null +++ b/App/rss_reader.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3.8 +import logging +from App.RSSListener import RSSListener +from App import Args_parser +from termcolor import colored +from App.Colors import Colors + + +def main(): + try: + args = Args_parser.parsing_args() + Args_parser.start_settings(args) + logging.info("Program launch") + rss_listener = RSSListener(args.limit, args.json, args.date, args.to_html, args.to_pdf) + rss_listener.start(args.source) + except Exception as e: + print(colored(str(e), Colors["error"])) + close_program() + + +def close_program(): + print(colored("The program suddenly completed its work", Colors["error"])) + exit() + + +if __name__ == '__main__': + main() diff --git a/Cache/.gitkeep b/Cache/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/Cache/.gitkeep @@ -0,0 +1 @@ + diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..404c98d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,4 @@ +FROM python:latest +COPY . /serverr/ +RUN pip3 install -r ./serverr/requirements.txt +WORKDIR /serverr \ No newline at end of file diff --git a/PythonHomework/__init__.py b/PythonHomework/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/PythonHomework/settings.py b/PythonHomework/settings.py new file mode 100644 index 0000000..0daa0ea --- /dev/null +++ b/PythonHomework/settings.py @@ -0,0 +1,125 @@ +""" +Django settings for PythonHomework project. + +Generated by 'django-admin startproject' using Django 2.2.7. + +For more information on this file, see +https://docs.djangoproject.com/en/2.2/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/2.2/ref/settings/ +""" + +import os + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/2.2/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = '-++fsq$(tcdmr-2n#%pybeg*4m=%rot8-_c02#!m*9g@h955(o' + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = [] + + +# Application definition + +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', +] + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'PythonHomework.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [os.path.join(BASE_DIR, 'templates')] + , + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'PythonHomework.wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/2.2/ref/settings/#databases + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'NAME': 'django_db', + 'USER': 'django_user', + 'PASSWORD': 'Sekret', + 'HOST': 'db', + 'PORT': '5432', + } +} + + +# Password validation +# https://docs.djangoproject.com/en/2.2/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/2.2/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/2.2/howto/static-files/ + +STATIC_URL = '/static/' diff --git a/PythonHomework/urls.py b/PythonHomework/urls.py new file mode 100644 index 0000000..c8d5183 --- /dev/null +++ b/PythonHomework/urls.py @@ -0,0 +1,21 @@ +"""PythonHomework URL Configuration + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/2.2/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" + +from django.conf.urls import url, include + +urlpatterns = [ + url(r'^', include('landing.urls')), +] diff --git a/PythonHomework/wsgi.py b/PythonHomework/wsgi.py new file mode 100644 index 0000000..507d571 --- /dev/null +++ b/PythonHomework/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for PythonHomework project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/2.2/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'PythonHomework.settings') + +application = get_wsgi_application() diff --git a/README.md b/README.md new file mode 100644 index 0000000..20129e6 --- /dev/null +++ b/README.md @@ -0,0 +1,49 @@ +# FinalTask + +## 1 iter + +Выполнена + +## 2 iter + +Выполнена + +## 3 iter + +Выполнена. +Для хранения данных использовал pickle. Данные храню +в папке Cache. Название файлов соответствуют дате статей, хранящихся там. +Отвечает за загрузку и выгрузку статей из файла класс Saver + +## 4 iter + +Выполнена +Данные сохраняю в формат html и pdf +В условиях не было указано в каком формате поставляется, поэтому выбрал для этого свой. +Путь должен содержать название файла и формат. +Примеры: +/home/ilya/snap/test.pdf +./test.html + + +## 5 iter + +Выполнена + +## 6 iter + +Частично выполнил +К сожалению не успел сдлеать из-за ряда причин. +Сразу извиняюсь за код в этом задании) +Установка: +Перейти в директорию с проектом и запустить команду: +sudo docker-compose up --build + +## Help +usage: rss-reader [-h] [--version] [--json] [--verbose] [--limit LIMIT] + [--date DATE] [--to_html TO_HTML] [--to_pdf TO_PDF] + [--colorize] + source + +## Install +python setup.py install diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..49033a9 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,19 @@ +version: "3" +services: + db: + image: "postgres:12.1" + container_name: "my_postgres" + environment: + - POSTGRES_USER=django_user + - POSTGRES_PASSWORD=Sekret + - POSTGRES_DB=django_db + expose: + - "5432" + server: + container_name: "server" + build: ./ + command: bash -c "python3 ./manage.py migrate && python3 ./manage.py runserver 0.0.0.0:2504" + ports: + - 2504:2504 + depends_on: + - db \ No newline at end of file diff --git a/landing/__init__.py b/landing/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/landing/forms.py b/landing/forms.py new file mode 100644 index 0000000..6601c67 --- /dev/null +++ b/landing/forms.py @@ -0,0 +1,6 @@ +from django import forms + + +class MainForm(forms.Form): + rss = forms.CharField(max_length=300) + limit = forms.IntegerField() \ No newline at end of file diff --git a/landing/urls.py b/landing/urls.py new file mode 100644 index 0000000..f8862d4 --- /dev/null +++ b/landing/urls.py @@ -0,0 +1,23 @@ +"""test_project URL Configuration + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/1.10/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: url(r'^$', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.conf.urls import url, include + 2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls')) +""" +from django.conf.urls import url, include +from django.contrib import admin +from landing import views + +urlpatterns = [ + url(r'^$', views.home, name='home'), + url(r'^news', views.news, name='news'), +] diff --git a/landing/views.py b/landing/views.py new file mode 100644 index 0000000..1aa8fc9 --- /dev/null +++ b/landing/views.py @@ -0,0 +1,25 @@ +from django.shortcuts import render +from .forms import MainForm +from App.RSSListener import RSSListener + + +def home(request): + submitbutton = request.POST.get("submit") + + rss = '' + limit = 20 + + form = MainForm(request.POST or None) + if form.is_valid(): + rss = form.cleaned_data.get("rss") + limit = form.cleaned_data.get("limit") + listener = RSSListener(limit, False, None, "./templates/news.html", None) + listener.start(rss) + + context = {'form': form, 'rss': rss, 'limit': limit, 'submitbutton': submitbutton} + + return render(request, 'landing.html', context) + + +def news(request): + return render(request, 'news.html', locals()) diff --git a/manage.py b/manage.py new file mode 100755 index 0000000..3f448eb --- /dev/null +++ b/manage.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" +import os +import sys + + +def main(): + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'PythonHomework.settings') + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == '__main__': + main() diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..0a4c126 --- /dev/null +++ b/setup.py @@ -0,0 +1,18 @@ +from setuptools import setup, find_packages +import App + +setup( + name="RSS Reader", + version=App.__version__, + description="Utility to process RSS", + author="Borodin Ilya", + author_email="ilya.borodin8@gmail.com", + url="https://github.com/ilyaborodin/PythonHomework/tree/Final_Task", + packages=find_packages(), + install_requires=['feedparser==5.2.1', 'weasyprint', 'termcolor'], + python_requires='>=3.8', + entry_points={ + "console_scripts": ["rss-reader=App.rss_reader:main"], + } + +) diff --git a/templates/landing.html b/templates/landing.html new file mode 100644 index 0000000..7e01388 --- /dev/null +++ b/templates/landing.html @@ -0,0 +1,44 @@ + + + + +