wesmiler

app/api/controller/TaskController.php

@@ -2,23 +2,75 @@
 
 namespace app\api\controller;
 
+use app\index\model\GoodsModel;
+use app\index\service\Goods;
 use app\index\service\Grab;
+use app\index\service\PRedis;
 use think\Controller;
 
 class TaskController extends Controller
 {
 
+    /**
+     * 新品数据抓取
+     */
     public function grabPublishGoods(){
-        ini_set('user_agent', 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; SV1; .NET CLR 1.1.4322)');
+        $result = Grab::instance()->grabData('nike_cn');
+        showJson(1001, 1004, ['count'=> count($result),'result'=> $result]);
+    }
+
+    /**
+     * 新品抓取数据处理
+     */
+    public function catchGrabGoods(){
         set_time_limit(0);
-        Grab::instance()->grabData('test');
+        $resource = input('r','nike_cn');
+        try {
+            $goodIds = [];
+            $goodsList = [];
+            $queenKey = "queens:grabs_{$resource}:goods_".date('YmdH');
+            for($i=0; $i<50; $i++){
+                $goodsInfo = PRedis::lpop($queenKey);
+                $goodsInfo = $goodsInfo? json_decode($goodsInfo, true) : [];
+                $productId = isset($goodsInfo['product_id'])? trim($goodsInfo['product_id']) : '';
+                $cacheKey = "task:grabs_{$resource}:goods:".$productId;
+                if($goodsInfo && !PRedis::get($cacheKey)){
+                    if(Goods::checkGoodsExists($productId)){
+                        PRedis::set($cacheKey, 1, 3600);
+                        continue;
+                    }
+
+                    $goodIds[] = $productId;
+                    $goodsList[] = $goodsInfo;
+                    PRedis::set($cacheKey, json_encode($goodsInfo, 256), 3600);
+                }
+            }
+
+            // 入库
+            if($goodsList){
+                $res = GoodsModel::insertAll($goodsList);
+                if(!$res){
+                    return showJson(1004, '处理抓取新品数据失败', "\n");
+                }
+            }
+
+            $msg = "更新新品抓取数据结果，累计抓取到" . count($goodIds) . "个商品数据";
+            return showJson(1005, $msg, "\n");
+        } catch (\Exception $exception) {
+            return showJson(1004, $exception->getMessage(), '', "\n");
+        }
 
-        showJson(1001, 1004);
     }
 
-    public function test(){
-        $ua = isset($_SERVER['HTTP_USER_AGENT'])? trim($_SERVER['HTTP_USER_AGENT']) : '';
-        var_dump($ua);
-        exit;
+    /**
+     * 清楚缓存
+     */
+    public function clearGrabCache(){
+        $resource = input('r','nike_cn');
+        PRedis::delByKeys("grab:grabs_{$resource}:*");
+        PRedis::delByKeys("task:grabs_{$resource}:*");
+
+        return showJson(1005, '清理抓取新品数据缓存成功', "\n");
+
     }
 }

app/index/model/GoodsModel.php

@@ -0,0 +1,18 @@
+<?php
+// +----------------------------------------------------------------------
+// | ThinkCMF [ WE CAN DO IT MORE SIMPLE ]
+// +----------------------------------------------------------------------
+// | Copyright (c) 2013-2019 http://www.thinkcmf.com All rights reserved.
+// +----------------------------------------------------------------------
+// | Licensed ( http://www.apache.org/licenses/LICENSE-2.0 )
+// +----------------------------------------------------------------------
+// | Author: 老猫 <thinkcmf@126.com>
+// +----------------------------------------------------------------------
+namespace app\index\model;
+
+use think\Model;
+
+class GoodsModel extends Model
+{
+
+}

app/index/service/Goods.php

@@ -0,0 +1,22 @@
+<?php
+/**
+ * 短信服务
+ * @author wesmiler
+ */
+
+namespace app\index\service;
+use \app\index\model\GoodsModel;
+class Goods
+{
+    /**
+     * 验证预售商品是否已存在
+     * @param $productId 预售商品原始ID
+     * @return bool|mixed
+     */
+    public static function checkGoodsExists($productId){
+        if(empty($productId)){
+            return false;
+        }
+        return GoodsModel::where(['product_id'=> $productId])->value('id');
+    }
+}

app/index/service/Grab.php

@@ -1,6 +1,7 @@
 <?php
 namespace app\index\service;
 
+use app\index\model\GoodsModel;
 use think\Config;
 
 class Grab
@@ -33,32 +34,79 @@ class Grab
      * 抓取数据
      * @param string $resource
      */
-    public function grabData($resource=''){
-        var_dump($this->urls);
+    public function grabData($resource='', $refresh = false){
+        set_time_limit(0);
         $urls = isset($this->urls[$resource])? $this->urls[$resource] : '';
         $url = isset($urls['url'])? trim($urls['url']) : '';
-        echo $url."\n";
         if($url){
-            $cookie = '';
-            if(file_exists("./logs/{$resource}.cookie")){
-                $cookie = file_get_contents("./logs/{$resource}.cookie");
+            $goodsList = $productIds = [];
+            $result = grabRequest($url, '');
+            $objects = isset($result['objects'])? $result['objects'] : [];
+            PRedis::set('grabs:'.$resource.':results:'.date('YmdH'), $result, 3600);
+            foreach($objects as $goods){
+                // 预售参数
+                $goodInfo = [];
+                $productInfo = isset($goods['productInfo'])? $goods['productInfo'] : [];
+                $goods = isset($productInfo[0])? $productInfo[0] : [];
+                $merchProduct = isset($goods['merchProduct'])? $goods['merchProduct'] : [];
+                $productId = isset($merchProduct['id'])? trim($merchProduct['id']) : '';
+
+                // 验证商品是否已经更新
+                if(!$productId || PRedis::get('grabs:'.$resource.':checkGoods:'.$productId)){
+                    continue;
+                }
+
+                $productIds[] = $productId;
+                $startDate = isset($merchProduct['commerceStartDate'])? trim($merchProduct['commerceStartDate']) : '';
+                $goodInfo['product_id'] = $productId;
+                $goodInfo['start_date'] = $startDate? date('Y-m-d H:i:s', strtotime($startDate)) : '';
+                $publishDate = isset($merchProduct['commercePublishDate'])? trim($merchProduct['commercePublishDate']) : '';
+                $goodInfo['publish_date'] = $publishDate? date('Y-m-d H:i:s', strtotime($publishDate)) : '';
+                $goodInfo['publish_type'] = isset($merchProduct['publishType'])? trim($merchProduct['publishType']) : '';
+                $status = isset($merchProduct['status'])? trim($merchProduct['status']) : '';
+                $goodInfo['status'] = strtolower($status) == 'active'? 1 : 3;
+
+                // 价格参数
+                $merchPrice = isset($goods['merchPrice'])? $goods['merchPrice'] : [];
+                $goodInfo['msrp'] = isset($merchPrice['msrp'])? floatval($merchPrice['msrp']) : 0.00;
+                $goodInfo['full_price'] = isset($merchPrice['fullPrice'])? floatval($merchPrice['fullPrice']) : 0.00;
+                $goodInfo['current_price'] = isset($merchPrice['currentPrice'])? floatval($merchPrice['currentPrice']) : 0.00;
+                $goodInfo['currency'] = isset($merchPrice['currency'])? trim($merchPrice['currency']) : 'CNY';
+                $goodInfo['discounted'] = isset($merchPrice['discounted'])? intval($merchPrice['discounted']) : 0;
+                $goodInfo['country'] = isset($merchPrice['country'])? $merchPrice['country'] : 'CN';
+
+
+                // 可用状态
+                $availability = isset($goods['availability'])? $goods['availability'] : [];
+                $goodInfo['available'] = isset($availability['available'])? intval($availability['available']) : 0;
+
+                // 商品信息
+                $productContent = isset($goods['productContent'])? $goods['productContent'] : [];
+                $goodInfo['colors_txt'] = isset($productContent['colorDescription'])? $productContent['colorDescription'] : '';
+                $colors = isset($productContent['colors'])? $productContent['colors'] : [];
+                $goodInfo['colors'] = is_array($colors) && $colors? serialize($colors) : '';
+                $goodInfo['slug'] = isset($productContent['slug'])? $productContent['slug'] : '';
+                $goodInfo['full_title'] = isset($productContent['fullTitle'])? $productContent['fullTitle'] : '';
+                $goodInfo['title'] = isset($productContent['title'])? $productContent['title'] : '';
+                $goodInfo['subtitle'] = isset($productContent['subtitle'])? $productContent['subtitle'] : '';
+                $goodInfo['description'] = isset($productContent['description'])? $productContent['description'] : '';
+                $goodInfo['pdpGeneral'] = isset($productContent['pdpGeneral'])? $productContent['pdpGeneral'] : '';
+
+                // 相册数据
+                $imgUrls = isset($goods['imageUrls'])? $goods['imageUrls'] : [];
+                $goodInfo['thumb'] = isset($imgUrls['productImageUrl'])? $imgUrls['productImageUrl'] : '';
+
+                $goodsList[] = $goodInfo;
+                PRedis::rpush("queens:grabs_{$resource}:goods_".date('YmdH'), json_encode($goodInfo, 256));
+                PRedis::expire("queens:grabs_{$resource}:goods_".date('YmdH'), 3 * 3600);
+                PRedis::set('grabs:'.$resource.':checkGoods:'.$productId, $goodInfo, 3*24*3600);
             }
-            $header = [
-                "User-Agent:Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36",
-                "Host:www.nike.com",
-                "Upgrade-Insecure-Requests:1",
-                "Accept:text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
-                "Content-Type:text/html; charset=utf-8",
-//                "Cookie:{$cookie}",
-            ];
-//            $content = file_get_contents($url);
-            $content = file_get_contents($url);
-//            $content = grabRequest($url, '','');
-            file_put_contents('./logs/'.$resource.'_'.date('YmdHis').'.html', $content);
 
-        }
+            PRedis::set('grabs:'.$resource.':goods:'.date('YmdH'), $goodsList, 7*24*3600);
 
+        }
 
+       return $productIds;
     }
 
 }

data/config/config.php

@@ -19,7 +19,7 @@ return [
         // 连接密码
         'password' => 'derkj&6688',
         // 数据库索引
-        'select' => 1,
+        'select' => 2,
         // 超时
         'timeout' => 60,
         // 有效期

public/themes/default/index/block/header.html

@@ -16,6 +16,5 @@
     <script src="__TMPL__/weixin/public/assets/lib/jquery.min.js?v={$version}"></script>
     <script src="__TMPL__/weixin/public/assets/lib/vue.min.js?v={$version}"></script>
     <script src="__TMPL__/weixin/public/assets/lib/jquery-weui.min.js?v={$version}"></script>
-    <script src="__TMPL__/weixin/public/assets/js/common.js?v={$version}"></script>
 </head>
 <body>

public/themes/default/index/index.html

@@ -1,6 +1,5 @@
-<include file="weixin@block:header"/>
-<title>活动主题</title>
-<link rel="stylesheet" href="__TMPL__/weixin/public/assets/css/activity.css?v={$version}">
+<include file="index@block:header"/>
+<title>抓取列表</title>
 <div id="app" >
         <div>
             <iframe src="http://www.nike.com/launch?s=upcoming" frameborder="0"></iframe>
@@ -10,5 +9,9 @@
     </script>
 
 </div>
-<script src="__TMPL__/weixin/public/assets/js/activity.js?v={$version}"></script>
-<include file="weixin@block:footer"/>
+<script>
+    /*$.load('https://www.nike.com/launch?s=upcoming', function(res){
+        console.log(res)
+    })*/
+</script>
+<include file="index@block:footer"/>

vendor/thinkcmf/cmf/src/common.php

@@ -2315,27 +2315,63 @@ function httpRequest($url, $data=[], $type='post', $dataType='array', $timeout=6
  * @param int $timeout
  * @return mixed
  */
-function grabRequest($url, $header=[], $data=[], $type='post', $timeout=60){
+function grabRequest($url, $header=[], $data=[], $type='get', $dataType='array', $timeout=60){
     $data = $data && is_array($data)? http_build_query($data) : '';
-    $url = strtolower($type) == 'get'? $url.(strpos($url, '?') === false ? '?' : ''). $data : $url;
+    $url = strtolower($type) == 'get'? $url.(strpos($url, '?') === false && $data? '?' : ''). $data : $url;
     $ch = curl_init($url);
     if($header){
         curl_setopt($ch, CURLOPT_HTTPHEADER, $header);
     }else{
         curl_setopt($ch, CURLOPT_HEADER, 0);
     }
-echo $url."<br>";exit;
-    curl_setopt($ch, CURLOPT_HEADER,0 );
+    
+    if (!empty($cookie)) {
+        curl_setopt($ch, CURLOPT_COOKIE, $cookie);
+    }
     curl_setopt($ch, CURLOPT_FRESH_CONNECT, 1);
     curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
     curl_setopt($ch, CURLOPT_FOLLOWLOCATION,1);
     curl_setopt($ch, CURLOPT_FORBID_REUSE, 1);
-//    curl_setopt($ch, CURLOPT_SSLVERSION, 3);
-    curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0); // 对认证证书来源的检查，0-规避ssl的证书检查
-    curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0);
+    curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); // 对认证证书来源的检查，0-规避ssl的证书检查
+    curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
     curl_setopt($ch, CURLOPT_TIMEOUT, $timeout);
     $ret = curl_exec($ch);
-    echo $ret;
     curl_close($ch);
+    if(strtolower($dataType) == 'array' && !is_array($ret)){
+        $ret = json_decode($ret, true);
+    }
     return $ret;
+}
+
+
+/**
+ *
+ * 接口请求
+ * @author wesmiler
+ * @param $url 接口地址
+ * @param $data
+ * @param $type
+ * @param int $timeout
+ * @return mixed
+ */
+function requestCookies($url)
+{
+    $ch = curl_init($url);
+    curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);    //禁止 cURL 验证对等证书
+    curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);    //是否检测服务器的域名与证书上的是否一致
+    curl_setopt($ch, CURLOPT_POST, 1);
+    curl_setopt($ch, CURLOPT_HEADER, 1);
+    curl_setopt($ch, CURLOPT_FRESH_CONNECT, 1);
+    curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
+    curl_setopt($ch, CURLOPT_FORBID_REUSE, 1);
+    curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0); // 对认证证书来源的检查，0-规避ssl的证书检查
+    curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0);
+    $ret = curl_exec($ch);
+    curl_close($ch);
+    file_put_contents('./logs/cookie.txt', $ret);
+    $preg_cookie = '/Set-Cookie: (.*?);/m';
+    if (preg_match_all($preg_cookie, $ret, $cookie)) {
+        $cookie = implode(';', $cookie['1']);
+    }
+    return $cookie;
 }