package cache



import (

	"server/service"

	"fmt"

	"github.com/go-redis/redis"

)



type Cache struct {

	conn *redis.Client

}



func (this *Cache) CacheInit() {

	this.connect()



}

func (this *Cache) connect() {

	conf := service.ReadRedisConfig()

	this.conn = redis.NewClient(&redis.Options{

		Addr:     conf.Addr,

		Password: conf.Password,

		DB:       conf.DB,

	})



}

func (this *Cache) Get(id string) (string, bool) {

	result, err := this.conn.Get(id).Result()

	if err != nil {

		fmt.Println(err)

		return "", false

	}

	return result, true

}



func (this *Cache) Set(id string, content string) bool {

	_, err := this.conn.Set(id, content, 0).Result()

	if err != nil {

		fmt.Println(err)

		return false

	} else {

		return true

	}

}



func (this *Cache) CheckUserExist(sign string) bool {

	result, err := this.conn.SIsMember("User", sign).Result()

	if err != nil {

		fmt.Println(err)

		return false

	}

	return result

}



func (this *Cache) AddUser(name string) bool {

	_, err := this.conn.SAdd("User", name).Result()

	if err != nil {

		fmt.Println(err)

		return false

	} else {

		return true

	}



}

上交所发布关于可转换公司债券适当性管理相关事项的通知，个人投资者参与向不特定对象发行的可转债申购、交易，应当同时符合下列条件：申请权限开通前20个交易日证券账户及资金账户内的资产日均不低于人民币10万元（不包括该投资者通过融资融券融入的资金和证券）；参与证券交易24个月以上。

get_history(5, frequency='1d', field='close', security_list=['123084.SZ'], fq=None, include=False, fill='nan')

APScheduler (3.3.1)

arch (3.2)

bcolz (1.2.1)

beautifulsoup4 (4.6.0)

bleach (1.5.0)

boto (2.43.0)

Bottleneck (1.0.0)

bz2file (0.98)

cachetools (3.1.0)

click (4.0)

contextlib2 (0.4.0)

crypto (1.4.1)

cvxopt (1.1.8)

cx-Oracle (8.0.1)

cycler (0.10.0)

cyordereddict (0.2.2)

Cython (0.22.1)

decorator (4.0.10)

entrypoints (0.2.2)

fastcache (1.0.2)

gensim (0.13.3)

h5py (2.6.0)

hmmlearn (0.2.0)

hs-udata (0.3.6)

html5lib (0.9999999)

ipykernel (4.5.0)

ipython (5.1.0)

ipython-genutils (0.1.0)

ipywidgets (5.2.2)

jieba (0.38)

Jinja2 (2.8)

jsonpickle (1.0)

jsonschema (2.5.1)

jupyter (1.0.0)

jupyter-client (4.4.0)

jupyter-console (5.0.0)

jupyter-core (4.2.0)

jupyter-kernel-gateway (1.1.1)

Keras (2.3.1)

Keras-Applications (1.0.8)

Keras-Preprocessing (1.1.0)

line-profiler (2.1.2)

Logbook (1.4.3)

lxml (4.5.0)

Markdown (2.2.0)

MarkupSafe (0.23)

matplotlib (1.5.3)

mistune (0.7.3)

Naked (0.1.31)

nbconvert (4.2.0)

nbformat (4.1.0)

networkx (1.9.1)

nose (1.3.6)

notebook (4.2.3)

numexpr (2.6.1)

numpy (1.11.2)

pandas (0.23.4)

patsy (0.4.0)

pexpect (4.2.1)

pickleshare (0.7.4)

pip (9.0.1)

pkgconfig (1.0.0)

prompt-toolkit (1.0.8)

protobuf (3.3.0)

ptvsd (2.2.0)

ptyprocess (0.5.1)

PyBrain (0.3)

pycrypto (2.6.1)

Pygments (2.1.3)

PyMySQL (0.9.3)

pyparsing (2.1.10)

python-dateutil (2.7.5)

pytz (2015.4)

PyWavelets (0.4.0)

PyYAML (5.3.1)

pyzmq (16.1.0.dev0)

qtconsole (4.2.1)

requests (2.7.0)

retrying (1.3.3)

scikit-learn (0.18)

scipy (0.18.0)

seaborn (0.7.1)

setuptools (28.7.1)

setuptools-scm (3.1.0)

shellescape (3.4.1)

simplegeneric (0.8.1)

simplejson (3.17.0)

six (1.10.0)

sklearn (0.0)

smart-open (1.3.5)

SQLAlchemy (1.0.8)

statsmodels (0.10.2)

TA-Lib (0.4.10)

tables (3.3.0)

tabulate (0.7.5)

tensorflow (1.3.0rc1)

tensorflow-tensorboard (0.1.2)

terminado (0.6)

Theano (0.8.2)

toolz (0.7.4)

tornado (4.4.2)

traitlets (4.3.1)

tushare (1.2.48)

tzlocal (1.3)

wcwidth (0.1.7)

Werkzeug (0.12.2)

wheel (0.29.0)

widgetsnbextension (1.2.6)

xcsc-tushare (1.0.0)

xgboost (0.6a2)

xlrd (1.1.0)

xlwt (1.3.0)

zipline (0.8.3)

You are using pip version 9.0.1, however version 22.1.2 is available.

You should consider upgrading via the 'pip install --upgrade pip' command.

在中国网络环境下获取Golang.org上的Golang Packages

背景

目前在中国网络环境下无法访问Golang.org。

问题

不能运行go get golang.org/x/XX来获取Golang packages。

解决方案

方案 A: 使用github 上的镜像



获取Golang Package在github镜像上的路径: golang.org/x/PATH_TO_PACKAGE --> github.com/golang/PATH_TO_PACKAGE.



   // Ex:

   golang.org/x/net/context --> github.com/golang/net/context

运行go get来安装github镜像的Golang packages。



   // Ex:

   go get github.com/golang/net/context

你会碰到如下错误提示:



   package github.com/golang/net/context:

   code in directory /go/src/github.com/golang/net/context

   expects import "golang.org/x/net/context"

忽略错误。 Golang的Package的源代码已经成功下载于：

$GOPATH/src/github.com/golang/PATH_TO_PACKAGE.



复制 $GOPATH/src/github.com/golang/PATH_TO_PACKAGE 到 $GOPATH/src/golang.org/x/PATH_TO_PACKAGE.



   // Ex:

   mkdir $GOPATH/src/golang.org/x -p

   cp $GOPATH/src/github.com/golang/net $GOPATH/src/golang.org/x/ -rf

运行 go build 来编译。



方案 B: 使用第三方网站 - https://gopm.io/download



输入包路径即可下载zip文件。

conn = pymysql.connect(host=host, port=port, user=user, password=password, db=db, charset='utf8',timeout=3)
 

import requests



def initialize(context):

    # 初始化策略

    g.security = "600570.SS"

    set_universe(g.security)



    

def handle_data(context, data):

    

    url='http://www.baidu.com'

    headers={'User-Agent':'Google'}

    try:

        r=requests.get(url,headers=headers)

        r.encoding='utf8'

        print(r.text)

    except Exception as e:

        print('error')

        print(e)

The rshim driver provides a way to access the rshim resources on the BlueField target from external host machine

<script type="text/javascript">

    var submitBTN = document.getElementById("url_update");

    submitBTN.onclick = function (event) {

        // 注意这里是 onclick 函数

        console.log("click");

        $.ajax({

            url: "/update-site1",

            type: "POST",

            data: {value:1},

            cache: false,

            contentType: "application/x-www-form-urlencoded",

            success: function (data) {

                console.log(data);



                if (data.code !== 0) {

                    alert('更新失败')

                } else {

                    new_str = data.res + " " + data.count

                    $("#txt_content").val(new_str);

                    alert('更新资源成功！')

                }

            },

            fail: function (data) {

                alert("更新失败!");

            }

        });



        $.ajax({

            url: "/update-site1",

            type: "POST",

            data: {value:2},

            cache: false,

            contentType: "application/x-www-form-urlencoded",

            success: function (data) {

                console.log(data);



                if (data.code !== 0) {

                    alert('更新失败')

                } else {

                    new_str = data.res + " " + data.count

                    $("#txt_content").val(new_str);

                    alert('更新资源成功！')

                }

            },

            fail: function (data) {

                alert("更新失败!");

            }

        })



    }

    

</script>

contentType: "application/x-www-form-urlencoded",

processData: false,

	router.POST("/update-site1", controllers.BaiduSite1)

func BaiduSite1(ctx *gin.Context) {

	if ctx.Request.Method == "POST" {



		value := ctx.PostForm("value")

		fmt.Println(value)

		int_value, err := strconv.Atoi(value)

		if err != nil {

			ctx.JSON(http.StatusOK, gin.H{

				"code":  1,

				"res":   "",

				"count": 0,

			})

			return

		}

		res, count := webmaster.PushProcess(1, int_value)

		ctx.JSON(http.StatusOK, gin.H{

			"code":  0,

			"res":   res,

			"count": count,

		})

	}

}

value := ctx.PostForm("value")

ctx.JSON(http.StatusOK, gin.H{

			"code":  0,

			"res":   res,

			"count": count,

			

原因：在setuptools 58之后的版本已经废弃了use_2to3

pip install setuptools==57.5.0

需求 conda python

默认情况下，1024以下的端口，只有root用户可以使用，例如想用python3启动一个在80端口上的web服务，只能切换到root用户再启动。如果想不用root用户启动，怎么办呢？



解决办法

记录一种测试了的办法：

Linux 内核从 2.6.24 版本开始就有了能力的概念，这使得普通用户也能够做只有超级用户才能完成的工作。



使用 setcap 命令让指定程序拥有绑定端口的能力，这样即使程序运行在普通用户下，也能够绑定到 1024 以下的特权端口上。



# 给指定程序设置 CAP_NET_BIND_SERVICE 能力

$ sudo setcap cap_net_bind_service=+eip /home/xda/miniconda3/bin/python3.9





注意赋给的程序目标不能是软链接

关于申诉，除了再一次播报申诉地址（http://ziyuan.baidu.com/feedback）外，我们给各位站长提个醒，在撰写申诉内容时应该尽量将问题描述具体，引用SEO爱好者痞子瑞的经验：“网站索引量异常，可以使用百度搜索资源平台（原百度站长平台）的索引量查询工具，一级一级的遍历一下自己网站的主要子域名或目录，以确定到底是哪个子域名或目录的索引量出现了异常。”“每个频道选取一些页面，在百度网页搜索中直接搜索这样页面的URL，以定位被删除快照网页的最小范围。”“然后在投诉内容中明确给出“病体”的URL，并附上相应的数据变动截图。”这样才便于处理投诉的百度工作人员快速寻找问题症结。

[url]https://release.axocdn.com/lin ... 1.deb[/url] 

[url]https://release.axocdn.com/lin ... 1.rpm[/url] 

[url]https://release.axocdn.com/lin ... ar.gz[/url] 

[url]https://release.axocdn.com/win ... 1.exe[/url] 

sudo dpkg -i gitkraken-amd64.deb

sudo apt --fix-broken install

from baidu_collection import baidu_site_collect

from seo_info import crawl_info

from configure.settings import DBSelector

import datetime

import argparse



client = DBSelector().mongo('qq')

doc = client['db_parker']['seo']





def main():



    parser = argparse.ArgumentParser()

    '''

    Command line options

    '''

    parser.add_argument(

        '-n',

        '--name', type=str,

        help='input web domain'

    )



    parser.add_argument(

        '-f',

        '--file', type=str,

        help='input web site domain file name'

    )



    FLAGS = parser.parse_args()

    site_list=

    if FLAGS.name:

        print(FLAGS.name)

        if '.' in FLAGS.name:

            site_list.append(FLAGS.name)



    elif FLAGS.file:

        print(FLAGS.file)

        with open(FLAGS.file,'r') as fp:

            webs=fp.readlines()



        site_list.extend(list(map(lambda x:x.strip(),webs)))



    if site_list:



        run(site_list=site_list)

    else:

        print("please input correct web domain")





def run(site_list):



    # TODO： 改为命令行形式





    for site in site_list:

        count = baidu_site_collect(site)

        info = crawl_info(site)

        print(info)

        print(count)

        info['site'] = site

        info['baidu_count'] = count

        info['update_time'] = datetime.datetime.now()

        doc.insert_one(info)





if __name__ == '__main__':

    main()

from parsel import Selector

import requests



def baidu_site_collect(site):

    # 百度收录

    headers = {'User-Agent': 'Chrome Google FireFox IE'}

    url = 'https://www.baidu.com/s?wd=site:{}&rsv_spt=1&rsv_iqid=0xf8b7b7e50006c034&issp=1&f=8&rsv_bp=1&rsv_idx=2&ie=utf-8&tn=baiduhome_pg&rsv_enter=0&rsv_dl=ib&rsv_sug3=14&rsv_sug1=7&rsv_sug7=100&rsv_n=2&rsv_btype=i&inputT=8238&rsv_sug4=8238'.format(site)

    resp = requests.get(

        url=url,

        headers=headers

    )



    resp.encoding='utf8'

    html = resp.text

    selector = Selector(text=html)



    count = selector.xpath('//div[@class="op_site_domain c-row"]/div/p/span/b/text()').extract_first()

    if count:

        count=int(count.replace(',',''))

    return count



if __name__=='__main__':

    site='30daydo.com'

    print(baidu_site_collect(site))

import argparse

from atexit import register

import sys

import requests

import re

from parsel import Selector



#参数自定义



# parser = argparse.ArgumentParser()

# parser.add_argument('-r', dest='read', help='path file')

# parser.add_argument('-u',dest='read',help='targetdomain')

# parser_args = parser.parse_args()

#爬虫模块查询



VERBOSE = True



def askurl(target_url):

    headers = {

        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.106 Safari/537.36'

    }





    #baidu权重

    baidu_url=f"https://rank.chinaz.com/{target_url}"

    baidu_txt=requests.get(url=baidu_url,headers=headers)

    baidu_html=baidu_txt.content.decode('utf-8')

    baidu_PC=re.findall('PC端</i><img src="//csstools.chinaz.com/tools/images/rankicons/baidu(.*?).png"></a></li>',baidu_html,re.S)

    baidu_moblie=re.findall('移动端</i><img src="//csstools.chinaz.com/tools/images/rankicons/bd(.*?).png"></a></li>',baidu_html,re.S)

    #分割线

    print("*"*60)



    #如果查询html中有正则出来到权重关键字就输出，否则将不输出

    result={}



    baidu_pc_weight = None

    baidu_mobile_weight = None



    if len(baidu_PC) > 0:

        print('百度_PC:', baidu_PC[0])

        baidu_pc_weight=baidu_PC[0]

    if len(baidu_moblie) > 0:

        print('百度_moblie:', baidu_moblie[0])

        baidu_mobile_weight = baidu_moblie[0]

    else:

        print("百度无权重")



    result['baidu_pc_weight']=baidu_pc_weight

    result['baidu_mobile_weight']=baidu_mobile_weight



    #360权重

    url=f"https://rank.chinaz.com/sorank/{target_url}/"

    text = requests.get(url=url,headers=headers)

    html=text.content.decode('utf-8')

    sorank360_PC=re.findall('PC端</i><img src="//csstools.chinaz.com/tools/images/rankicons/360(.*?).png"></a><',html,re.S)

    sorank360_Mobile=re.findall('移动端</i><img src="//csstools.chinaz.com/tools/images/rankicons/360(.*?).png"',html,re.S)



    _360_pc_weight=None

    _360_mobile_weight=None



    # 如果查询html中有正则出来到权重关键字就输出，否则将不输出

    if len(sorank360_PC) > 0:

        _360_pc_weight=sorank360_PC[0]

        print("360_PC:", sorank360_PC[0])

    if len(sorank360_Mobile) > 0:

        _360_mobile_weight=sorank360_Mobile[0]

        print("360_moblie:", sorank360_Mobile[0])

    else:

        print("360无权重")



    result['360_pc_weight']=_360_pc_weight

    result['360_mobile_weight']=_360_mobile_weight





    #搜狗权重





    sogou_pc_weight=None

    sogou_mobile_weight=None



    sogou_url = f"https://rank.chinaz.com/sogoupc/{target_url}"

    sougou_txt = requests.get(url=sogou_url, headers=headers)

    sougou_html = sougou_txt.content.decode('utf-8')

    sougou_PC = re.findall('PC端</i><img src="//csstools.chinaz.com/tools/images/rankicons/sogou(.*?).png"></a></li>',sougou_html, re.S)

    sougou_mobile = re.findall('移动端</i><img src="//csstools.chinaz.com/tools/images/rankicons/sogou(.*?).png"></a></li>',sougou_html, re.S)



    # 如果查询html中有正则出来到权重关键字就输出，否则将不输出

    if len(sougou_PC) > 0:

        print('搜狗_PC：', sougou_PC[1])

        sogou_pc_weight=sougou_PC[1]

        

    if len(sougou_mobile) > 0 :

        print('搜狗_moblie：', sougou_mobile[1])

        sogou_mobile_weight=sougou_mobile[1]



    else:

        print('搜狗无权重')





    result['sogou_pc_weight']=sogou_pc_weight

    result['sogou_mobile_weight']=sogou_mobile_weight





    #神马权重

    shenma_pc_weight =None   

    shenma_url=f'https://rank.chinaz.com/smrank/{target_url}'

    shenma_txt=requests.get(url=shenma_url,headers=headers)

    shenma_html=shenma_txt.content.decode('utf-8')

    shenma_PC=re.findall('class="tc mt5"><img src="//csstools.chinaz.com/tools/images/rankicons/shenma(.*?).png"></a></li>',shenma_html,re.S)



    # 如果查询html中有正则出来到权重关键字就输出，否则将不输出

    if len(shenma_PC) > 0:

        print('神马权重为：', shenma_PC[1])

        shenma_pc_weight=shenma_PC[1]

    else:

        print("神马无权重")





    result['shenma_pc_weight']=shenma_pc_weight

    # result['shenma_mobile_weight']=None





    #头条权重



    toutiao_pc_weight=None

    toutiao_url=f'https://rank.chinaz.com/toutiao/{target_url}'

    toutiao_txt=requests.get(url=toutiao_url,headers=headers)

    toutiao_html=toutiao_txt.content.decode('utf-8')

    toutiao_PC=re.findall('class="tc mt5"><img src="//csstools.chinaz.com/tools/images/rankicons/toutiao(.*?).png"></a></li>',toutiao_html,re.S)



    # 如果查询html中有正则出来到权重关键字就输出，否则将不输出

    if len(toutiao_PC) > 0:

        print('头条权重为：', toutiao_PC[1])

        toutiao_pc_weight=toutiao_PC[1]

    else:

        print("头条无权重")



    result['toutiao_pc_weight']=toutiao_pc_weight

    # result['toutiao_mobile_weight']=None





    #备案信息、title、企业性质

    beian_url=f"https://seo.chinaz.com/{target_url}"

    beian_txt=requests.get(url=beian_url,headers=headers)

    beian_html=beian_txt.content.decode('utf-8')

    

    with open('beian_html.html','w') as fp:

        fp.write(beian_html)



    title,beian_no,name,ip,nature,register,years=parse_info(beian_html)

    

    result['name']=name

    result['title']=title

    result['beian_no']=beian_no

    result['ip']=ip

    result['nature']=nature

    result['register']=register

    result['years']=years





    try:

        print("备案信息:",beian_no,"名称:",name,"网站首页Title：",title,"企业性质：",nature,"IP地址为：",ip)

        print("*"*60)

    except:

        print("没有查询到有效信息！")



    return result



strip_fun = lambda x:x.strip() if x is not None else ""



def parse_info(html):



    resp = Selector(text=html)

    title = strip_fun(resp.xpath('//div[@class="_chinaz-seo-t2l ellipsis"]/text()').extract_first())

    table = resp.xpath('//table[@class="_chinaz-seo-newt"]/tbody')

    

    if table[0].xpath('.//tr[4]/td[2]/span[1]/i'):

        beian_num=strip_fun(table[0].xpath('.//tr[4]/td[2]/span[1]/i/a/text()').extract_first())

    else:

        beian_num=strip_fun(table[0].xpath('.//tr[4]/td[2]/span[1]/a/text()').extract_first())



    name=strip_fun(table[0].xpath('.//tr[4]/td[2]/span[2]/i/text()').extract_first())

    if not name:

        print('---->',name)

        name=strip_fun(table[0].xpath('.//tr[4]/td[2]/span[2]/i/a/text()').extract_first())



    nature=strip_fun(table[0].xpath('.//tr[4]/td[2]/span[3]/i/text()').extract_first())

    ip=strip_fun(table[0].xpath('.//tr[5]/td[2]/div/span[1]/i/a/text()').extract_first())

    register=strip_fun(table[0].xpath('.//tr[3]/td[2]/div[1]/span[1]/i/text()').extract_first())

    years=strip_fun(table[0].xpath('.//tr[3]/td[2]/div[2]/span[1]/i/text()').extract_first())







    return title,beian_num,name,ip,nature,register,years





def crawl_info(site):

    return askurl(site)



if __name__ == '__main__':

    main()

Got recursion not available from 8.8.8.8