init

pinterest/items.py

@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+
+# Define here the models for your scraped items
+#
+# See documentation in:
+# https://docs.scrapy.org/en/latest/topics/items.html
+
+import scrapy
+
+
+class PinterestItem(scrapy.Item):
+    # define the fields for your item here like:
+    note_detail_img = scrapy.Field()
+    note_detail = scrapy.Field()
+    note_detail_url = scrapy.Field()

pinterest/main.py

@@ -0,0 +1,9 @@
+__author__='pasaulis'
+
+from scrapy.cmdline import execute
+import sys
+import os
+
+sys.path.append(os.path.dirname(os.path.abspath(__file__)))
+
+execute(["scrapy","crawl","pinter"])

pinterest/middlewares.py

@@ -0,0 +1,103 @@
+# -*- coding: utf-8 -*-
+
+# Define here the models for your spider middleware
+#
+# See documentation in:
+# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+
+from scrapy import signals
+
+
+class PinterestSpiderMiddleware(object):
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the spider middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_spider_input(self, response, spider):
+        # Called for each response that goes through the spider
+        # middleware and into the spider.
+
+        # Should return None or raise an exception.
+        return None
+
+    def process_spider_output(self, response, result, spider):
+        # Called with the results returned from the Spider, after
+        # it has processed the response.
+
+        # Must return an iterable of Request, dict or Item objects.
+        for i in result:
+            yield i
+
+    def process_spider_exception(self, response, exception, spider):
+        # Called when a spider or process_spider_input() method
+        # (from other spider middleware) raises an exception.
+
+        # Should return either None or an iterable of Request, dict
+        # or Item objects.
+        pass
+
+    def process_start_requests(self, start_requests, spider):
+        # Called with the start requests of the spider, and works
+        # similarly to the process_spider_output() method, except
+        # that it doesn’t have a response associated.
+
+        # Must return only requests (not items).
+        for r in start_requests:
+            yield r
+
+    def spider_opened(self, spider):
+        spider.logger.info('Spider opened: %s' % spider.name)
+
+
+class PinterestDownloaderMiddleware(object):
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the downloader middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_request(self, request, spider):
+        # Called for each request that goes through the downloader
+        # middleware.
+
+        # Must either:
+        # - return None: continue processing this request
+        # - or return a Response object
+        # - or return a Request object
+        # - or raise IgnoreRequest: process_exception() methods of
+        #   installed downloader middleware will be called
+        return None
+
+    def process_response(self, request, response, spider):
+        # Called with the response returned from the downloader.
+
+        # Must either;
+        # - return a Response object
+        # - return a Request object
+        # - or raise IgnoreRequest
+        return response
+
+    def process_exception(self, request, exception, spider):
+        # Called when a download handler or a process_request()
+        # (from other downloader middleware) raises an exception.
+
+        # Must either:
+        # - return None: continue processing this exception
+        # - return a Response object: stops process_exception() chain
+        # - return a Request object: stops process_exception() chain
+        pass
+
+    def spider_opened(self, spider):
+        spider.logger.info('Spider opened: %s' % spider.name)

pinterest/pipelines.py

@@ -0,0 +1,93 @@
+# -*- coding: utf-8 -*-
+
+# Define your item pipelines here
+#
+# Don't forget to add your pipeline to the ITEM_PIPELINES setting
+# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
+
+import json
+import random
+import os
+
+import oss2
+import  requests
+import hashlib
+import urllib.parse
+import logging
+logger = logging.getLogger(__name__)
+from scrapy.utils.project import  get_project_settings
+
+class PinterestPipeline(object):
+
+    def __init__(self):
+        #self.fp = open('log.json','a',encoding='utf-8')
+        self.setting = get_project_settings();
+        self.headers = {}
+        self.headers['Content-Type'] = 'application/json'
+        #获取java服务器token
+        token = self.request_java_login();
+        self.headers['token'] = token
+
+    def process_item(self, item, spider):
+       # item_json = json.dumps(dict(item), ensure_ascii = False)
+       # self.fp.write(item_json+'\n\n')
+
+        logger.info("******************* 原始 Item 数据 BEGIN **************************")
+        logger.info(json.dumps(dict(item), ensure_ascii = False))
+        logger.info("******************* 原始 Item 数据 END **************************")
+
+        item['note_detail_img'] = self.oss_upload( item['note_detail_img'])
+
+        self.request_java_api(item);
+        return item
+
+    #替换图片地址
+    def oss_upload(self , img_url):
+
+        img_file_obj = os.path.splitext(img_url);
+        img_file_name = hashlib.md5(img_file_obj[0].encode('utf-8')).hexdigest()
+        img_url_path = '%s/%s%s'%(random.randint(1,10),img_file_name,img_file_obj[1])
+
+        auth = oss2.Auth(self.setting['OSS']['key'] , self.setting['OSS']['secret'])
+        bucket = oss2.Bucket(auth, self.setting['OSS']['bucket_url'], self.setting['OSS']['bucket_name'])
+        input = requests.get(img_url)
+        bucket.put_object(img_url_path, input)
+
+        ret = bucket.sign_url('GET', img_url_path , 60)
+
+        return urllib.parse.unquote(ret[0:ret.find('?')])
+    
+    def request_java_api(self , item):
+
+        #请求 JAVA 服务器接口 传输 数据
+        data_json = json.dumps({
+                "auditStatus": 0,
+                "nickName": "Alain",
+                "noteContent":item['note_detail'],
+                "noteContentHtml": item['note_detail'],
+                "noteLabelRelationList": [],
+                "noteMediaParamList": [
+                    {
+                        "mediaType": 1,
+                        "mediaUrl": item['note_detail_img']
+                    }
+                ],
+                "noteTitle": item['note_detail'],
+                "url": item['note_detail_url'],
+                "userId": 182
+            })
+
+        requests.post("https://youyu-admin-api.5fire.tech/admin/user_note/release_note?%d"%random.randint(1,10) , data_json, headers=self.headers)
+
+        logger.info("******************* 请求接口 Item 数据 BEGIN **************************")
+        logger.info(json.dumps(dict(item), ensure_ascii = False))
+        logger.info("******************* 请求接口 Item 数据 END **************************")
+
+    def request_java_login(self):
+        ret = requests.get(url = "https://youyu-admin-api.5fire.tech/admin/v1/oauth/token?userName=admin&password=superman",headers=self.headers);
+
+        return ret.json()['data']['token'];
+
+    def close_spider(self,spider):
+        #self.fp.close()
+        pass

pinterest/run.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+cat search_word.txt | while read line
+do
+	#命令行内容
+	time2=$(date "+%Y%m%d%H%M%S")
+	echo $line
+	echo $time2
+	sed -i "s/SEARCH_WORD/$line/g" /root/spiderwork/pinterest/pinterest/spiders/pinter.py
+	echo 'start .............")' 
+        scrapy crawl pinter
+	echo 'end.............' 
+	sed -i "s/$line/SEARCH_WORD/g" /root/spiderwork/pinterest/pinterest/spiders/pinter.py
+done

pinterest/search_word.txt

@@ -0,0 +1,14 @@
+Women
+Home Renovation 
+Design 
+Weddings 
+Travel 
+books
+Architecture
+Fitness
+Food and drink
+Men outfits
+Home decor
+fashion
+Motorcycles
+Men accessories

pinterest/settings.py

@@ -0,0 +1,110 @@
+# -*- coding: utf-8 -*-
+
+# Scrapy settings for pinterest project
+#
+# For simplicity, this file contains only settings considered important or
+# commonly used. You can find more settings consulting the documentation:
+#
+#     https://docs.scrapy.org/en/latest/topics/settings.html
+#     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
+#     https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+
+BOT_NAME = 'pinterest'
+
+SPIDER_MODULES = ['pinterest.spiders']
+NEWSPIDER_MODULE = 'pinterest.spiders'
+
+#Setting log level
+LOG_LEVEL = "WARNING"
+
+#Setting log Path
+LOG_FILE = "./log.log"
+
+
+# Crawl responsibly by identifying yourself (and your website) on the user-agent
+USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.87 Safari/537.36'
+
+# Obey robots.txt rules
+ROBOTSTXT_OBEY = False
+
+# Configure maximum concurrent requests performed by Scrapy (default: 16)
+#CONCURRENT_REQUESTS = 32
+
+# Configure a delay for requests for the same website (default: 0)
+# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
+# See also autothrottle settings and docs
+#DOWNLOAD_DELAY = 3
+# The download delay setting will honor only one of:
+#CONCURRENT_REQUESTS_PER_DOMAIN = 16
+#CONCURRENT_REQUESTS_PER_IP = 16
+
+# Disable cookies (enabled by default)
+#COOKIES_ENABLED = True
+
+# Disable Telnet Console (enabled by default)
+#TELNETCONSOLE_ENABLED = False
+
+# Override the default request headers:
+DEFAULT_REQUEST_HEADERS = {
+   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+   'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
+   'upgrade-insecure-requests': 1,
+   #'x-app-version': '4d69f6e',
+   #'x-csrftoken': '14852c984af740bec57eca493cc83eb6',
+   #'x-pinterest-appstate': 'background',
+   #'x-pinterest-experimenthash': '048e9abb2302538ec11dc3a64a02d855810c77475e19eb8d5e02e943222a1279ff3c8777a2dee48397b0871082fa3e22b1249cdfd2b5b4ef8b307a45f9dc9162',
+   #'x-requested-with': 'XMLHttpRequest'
+}
+
+# Enable or disable spider middlewares
+# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+#SPIDER_MIDDLEWARES = {
+#    'pinterest.middlewares.PinterestSpiderMiddleware': 543,
+#}
+
+# Enable or disable downloader middlewares
+# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
+#DOWNLOADER_MIDDLEWARES = {
+#    'pinterest.middlewares.PinterestDownloaderMiddleware': 543,
+#}
+
+# Enable or disable extensions
+# See https://docs.scrapy.org/en/latest/topics/extensions.html
+#EXTENSIONS = {
+#    'scrapy.extensions.telnet.TelnetConsole': None,
+#}
+
+# Configure item pipelines
+# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
+ITEM_PIPELINES = {
+    'pinterest.pipelines.PinterestPipeline': 300,
+}
+
+# Enable and configure the AutoThrottle extension (disabled by default)
+# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
+#AUTOTHROTTLE_ENABLED = True
+# The initial download delay
+#AUTOTHROTTLE_START_DELAY = 5
+# The maximum download delay to be set in case of high latencies
+#AUTOTHROTTLE_MAX_DELAY = 60
+# The average number of requests Scrapy should be sending in parallel to
+# each remote server
+#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
+# Enable showing throttling stats for every response received:
+#AUTOTHROTTLE_DEBUG = False
+
+# Enable and configure HTTP caching (disabled by default)
+# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
+#HTTPCACHE_ENABLED = True
+#HTTPCACHE_EXPIRATION_SECS = 0
+#HTTPCACHE_DIR = 'httpcache'
+#HTTPCACHE_IGNORE_HTTP_CODES = []
+#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
+
+#OSS Config
+OSS = {
+    'key' : 'LTAIV6k7WdEKABig',
+    'secret': 't3rAS26DiSM6Z3dkUFCdTKqZBLUFCm',
+    'bucket_name': 'mrsq',
+    'bucket_url':'http://oss-cn-beijing.aliyuncs.com'
+}

pinterest/spiders/__init__.py

@@ -0,0 +1,4 @@
+# This package will contain the spiders of your Scrapy project
+#
+# Please refer to the documentation for information on how to create and manage
+# your spiders.

pinterest/spiders/ranslator.py

@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+import scrapy
+from googletrans import Translator
+translator = Translator()
+
+
+class RanslatorSpider(scrapy.Spider):
+    name = 'ranslator'
+    allowed_domains = ['baidu.com']
+    start_urls = ['http://baidu.com/']
+
+    def parse(self, response):
+        print(translator.translate('⚡CHECK OUR STORE ⚡💀👉 @www.brapwrap.com motorcycle Gear | T- Shirt | Rings | Bracelets | & More 🔥 🌐Free Shipping Worldwide 🌐',dest='zh-CN').text)

scrapy.cfg

@@ -0,0 +1,11 @@
+# Automatically created by: scrapy startproject
+#
+# For more information about the [deploy] section see:
+# https://scrapyd.readthedocs.io/en/latest/deploy.html
+
+[settings]
+default = pinterest.settings
+
+[deploy]
+#url = http://localhost:6800/
+project = pinterest