Here, we will see how we can upgrade our code to work with tensorflow 2.0.



This error is usually faced when we are loading pre-trained model with tensorflow session/graph or we are building flask api over a pre-trained model and loading model in tensorflow graph to avoid collision of sessions while application is getting multiple requests at once or say in case of multi-threadinng



After tensorflow 2.0 upgrade, i also started facing above error in one of my project when i had built api of pre-trained model with flask. So i looked around in tensorflow 2.0 documents to find a workaround, to avoid this runtime error and upgrade my code to work with tensorflow 2.0 as well rather than downgrading it to tensorflow 1.x .



I had a project on which i had written tutorial as well on how to build Flask api on trained keras model of text classification and then use it in production



But this project was not working after tensorflow upgrade and was facing runtime error.



Stacktrace of error was something like below:



 File "/Users/Upasana/Documents/playground/deploy-keras-model-in-production/src/main.py", line 37, in model_predict

    with backend.get_session().graph.as_default() as g:

  File "/Users/Upasana/Documents/playground/deploy-keras-model-in-production/venv-tf2/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py", line 379, in get_session

    '`get_session` is not available '

RuntimeError: `get_session` is not available when using TensorFlow 2.0.

Related code to get model

with backend.get_session().graph.as_default() as g:

        model = SentimentService.get_model1()

Related code to load model

def load_deep_model(self, model):

        json_file = open('./src/mood-saved-models/' + model + '.json', 'r')

        loaded_model_json = json_file.read()

        loaded_model = model_from_json(loaded_model_json)



        loaded_model.load_weights("./src/mood-saved-models/" + model + ".h5")



        loaded_model._make_predict_function()

        return loaded_model

get_session is removed in tensorflow 2.0 and hence not available.



so, in order to load saved model we switched methods. Rather than using keras’s load_model, we used tensorflow to load model so that we can load model using distribution strategy.



Note

The tf.distribute.Strategy API provides an abstraction for distributing your training across multiple processing units.



New code to get model

another_strategy = tf.distribute.MirroredStrategy()

with another_strategy.scope():

    model = SentimentService.get_model1()

New code to load model

def load_deep_model(self, model):

        loaded_model = tf.keras.models.load_model("./src/mood-saved-models/"model + ".h5")

        return loaded_model

This worked and solved the problem with runtime error of get_session not available in tensorflow 2.0 . You can refer to Tensorflow 2.0 upgraded article too



Hope, this will solve your problem too. Thanks for following this article.

import xml.etree.ElementTree as ET

import pickle

import os

from os import listdir, getcwd

from os.path import join









sets = [('2012', 'train'), ('2012', 'val'), ('2007', 'train'), ('2007', 'val'), ('2007', 'test')]



# 20类

classes = ["aeroplane", "bicycle", "bird", "boat", "bottle", "bus", "car", "cat", "chair", "cow", "diningtable", "dog",

           "horse", "motorbike", "person", "pottedplant", "sheep", "sofa", "train", "tvmonitor"]



# size w,h

# box x-min,x-max,y-min,y-max

def convert(size, box):

    dw = 1. / size[0]

    dh = 1. / size[1]

    x = (box[0] + box[1]) / 2.0 # 中心点位置

    y = (box[2] + box[3]) / 2.0

    w = box[1] - box[0]

    h = box[3] - box[2]

    x = x * dw

    w = w * dw

    y = y * dh

    h = h * dh # 全部转化为相对坐标

    return (x, y, w, h)





def convert_annotation(year, image_id):

    # 找到2个同样的文件

    in_file = open('VOCdevkit/VOC%s/Annotations/%s.xml' % (year, image_id))

    out_file = open('VOCdevkit/VOC%s/labels/%s.txt' % (year, image_id), 'w')



    tree = ET.parse(in_file)

    root = tree.getroot()

    size = root.find('size')

    w = int(size.find('width').text)

    h = int(size.find('height').text)



    for obj in root.iter('object'):

        difficult = obj.find('difficult').text

        cls = obj.find('name').text

        if cls not in classes or int(difficult) == 1: # difficult ==1 的不要了

            continue

        cls_id = classes.index(cls) # 排在第几位

        xmlbox = obj.find('bndbox')

        b = (float(xmlbox.find('xmin').text), float(xmlbox.find('xmax').text), float(xmlbox.find('ymin').text),

             float(xmlbox.find('ymax').text))

        # 传入的是w,h 与框框的周边

        bb = convert((w, h), b)

        out_file.write(str(cls_id) + " " + " ".join([str(a) for a in bb]) + '\n')





wd = getcwd()



for year, image_set in sets:

    # ('2012', 'train') 循环5次

    # 创建目录 一次性

    if not os.path.exists('VOCdevkit/VOC%s/labels/' % (year)):

        os.makedirs('VOCdevkit/VOC%s/labels/' % (year))



    # 图片的id数据

    image_ids = open('VOCdevkit/VOC%s/ImageSets/Main/%s.txt' % (year, image_set)).read().strip().split()



    # 结果写入这个文件

    list_file = open('%s_%s.txt' % (year, image_set), 'w')



    for image_id in image_ids:

        list_file.write('%s/VOCdevkit/VOC%s/JPEGImages/%s.jpg\n' % (wd, year, image_id)) # 补全路径

        convert_annotation(year, image_id)

    list_file.close()

package main import "fmt" func main() {

/* 这是我的第一个简单的程序 */    

fmt.Println("Hello, World!") 

}

Last updated August 21, 2016

In an effort to offer continued support beyond publication, we have listed many updates to code due to version updates.



[code - omission] Page 18



The command to start the "mailer" is missing a line. Where the book reads:

docker run -d \

  --name mailer \

the proper command should read:

docker run -d \

  --name mailer \

  dockerinaction/ch2_mailer

  

[code - regression] Page 68



Newer versions of Docker have changed the structure of the JSON returned by the docker inspect subcommand. If the following command does not work then use the replacement. Original:

docker inspect --format "{{json .Volumes}}" reader

Replacement:

docker inspect --format "{{json .Mounts}}" reader



[code - regression] Page 69



Newer versions of Docker have changed the structure of the JSON returned by the docker inspect subcommand. If the following command does not work then use the replacement. Original:

docker inspect --format "{{json .Volumes}}" student

Replacement:

docker inspect --format "{{json .Mounts}}" student



[code - regression] Page 74



The alpine image entrypoint has changed since original publication and has been unset. The last command on the page should now read:

docker run --rm \

  --volumes-from tools \

  --entrypoint /bin/sh \

  alpine:latest \

  -c 'ls /operations/*'

[code - regression] Page 75



The docker exec example on the top of page 75 was printed with the wrong tool name. The correct command is:

docker exec important_application /operations/tools/diagnostics



[code - regression] Page 86



It appears that nslookup behavior in the alpine image has changed. To run the example use the busybox:1 image.

docker run --rm \

  --hostname barker \

  busybox:1 \

  nslookup barker

  

[code - regression] Page 87 (top)



It appears that nslookup behavior in the alpine image has changed. To run the example use the busybox:1 image.

docker run --rm \

  --dns 8.8.8.8 \

  busybox:1 \

  nslookup docker.com

  

[code - regression] Page 87 (bottom)



It appears that nslookup behavior in the alpine image has changed. To run the example use the busybox:1 image.

docker run --rm \

  --dns-search docker.com \

  busybox:1 \

  nslookup registry.hub

  

[code - regression] Page 88 (bottom)



It appears that nslookup behavior in the alpine image has changed. To run the example use the busybox:1 image.

docker run --rm \

  --add-host test:10.10.10.255 \

  busybox:1 \

  nslookup test

  

[code - regression] Page 106



There are a few new problems with this example. First, the named repository (dockerfile/mariadb) no longer exists. You can use mariadb:5.5 as replacement. However, the second problem is that containers created from the mariadb image perform certain initialization at startup. That initialization work requires certain capabilities and to be started with the default user. The system should instead drop permissions after the initialization work is complete. Note that the real value of this example is in demonstrating different resource isolation mechanisms. It is not so important that you get it working. You can start the database with the following command:

docker run -d --name ch6_mariadb \

  --memory 256m \

  --cpu-shares 1024 \

  --cap-drop net_raw \

  -e MYSQL_ROOT_PASSWORD=test \

  mariadb:5.5

  

[code - regression] Page 107



Containers created from the wordpress:4.1 image perform certain initialization at startup and expect certain environment variables. That initialization work requires certain capabilities and to be started with the default user. The system should instead drop permissions after the initialization work is complete. Note that the real value of this example is in demonstrating different resource isolation mechanisms. It is not so important that you get it working. You can start wordpress with the following command:

docker run -d -P --name ch6_wordpress \

  --memory 512m \

  --cpu-shares 512 \

  --cap-drop net_raw \

  -e WORDPRESS_DB_PASSWORD=test \

  mariadb:5.5

  

[code - typo] Page 109



The device access example is missing the "run" subcommand. The command listed as:

docker -it --rm \

  --device /dev/video0:/dev/video0 \

  ubuntu:latest ls -al /dev

should have been written as:

docker run -it --rm \

  --device /dev/video0:/dev/video0 \

  ubuntu:latest ls -al /dev

  

[code - typo] Page 110 - 111



Several commands are missing the "run" subcommand. In each case the command begins with

docker -d ...

and should have been written as:

docker run -d ...



[code - regression] Page 115 (bottom)



The busybox and alpine images have been updated to fix the problem described in the paragraph below. The 'su' command does not have the SUID bit set and will not provide any automatic privilege escalation.



[command correction] Page 116



Boot2Docker has been discontinued and rolled into a newer project called Docker Machine. Because a reader is unlikely to have the boot2docker command installed, the command at the top of this page should be changed from:

boot2docker ssh

to the Docker Machine equivalent:

docker-machine ssh default

where default is the name of the machine you created.



[code - regression] Page 119



The ifconfig command has since been removed from ubuntu:latest. Instead of using the ubuntu:latest image for these examples use ubuntu:trusty. The example using ifconfig should look like:

docker run --rm \

  --privileged \

  ubuntu:trusty ifconfig

[Illustration Mistake] Page 136

Image layer ced2 on the left side of the illustration is listed at c3d2 on the right side. These two layers should represent the same item.



[code - typo] Page 140



Containers need not be in a running state in order to export their file system. The first command on page 140 uses the "run" subcommand but the command listed will never be able to start. Replace "run" with "create." The command should appear as follows:

docker create --name export-test \

  dockerinaction/ch7_packed:latest ./echo For Export

  

[code - missing line] Page 146



In the example Dockerfile near the top of the page the line with the RUN directive is missing part of the instruction. That line should read:

RUN apt-get update && apt-get install -y git



[code - evolution] Page 215



The registry:2 configuration file now requires the population of additional fields under "maintenance > uploadpurging." The example should currently look like:

# Filename: s3-config.yml

version: 0.1

log:

    level: debug

    fields:

        service: registry

        environment: development

storage:

    cache:

        layerinfo: inmemory

    s3:

        accesskey: 

        secretkey: 

        region: 

        bucket: 

        encrypt: true

        secure: true

        v4auth: true

        chunksize: 5242880

        rootdirectory: /s3/object/name/prefix

    maintenance:

        uploadpurging:

            enabled: true

            age: 168h

            interval: 24h

            dryrun: false

        readonly:

            enabled: false

http:

    addr: :5000

    secret: asecretforlocaldevelopment

    debug:

        addr: localhost:5001

        

[code - evolution] Page 216



The registry:2 configuration file now requires the population of additional fields under "maintenance > uploadpurging." The example should currently look like:

# Filename: rados-config.yml

version: 0.1

log:

    level: debug

    fields:

        service: registry

        environment: development

storage:

    cache:

        layerinfo: inmemory

    rados:

        poolname: radospool

        username: radosuser

        chunksize: 4194304

    maintenance:

        uploadpurging:

            enabled: false

            age: 168h

            interval: 24h

            dryrun: false

        readonly:

            enabled: false

http:

    addr: :5000

    secret: asecretforlocaldevelopment

    debug:

        addr: localhost:5001

        

[code - evolution] Page 218



The registry:2 configuration file now requires the population of additional fields under "maintenance > uploadpurging." The example should currently look like:

# Filename: redis-config.yml

version: 0.1

log:

    level: debug

    fields:

        service: registry

        environment: development

http:

    addr: :5000

    secret: asecretforlocaldevelopment

    debug:

        addr: localhost:5001

storage:

    cache:

        blobdescriptor: redis

    s3:

        accesskey: 

        secretkey: 

        region: 

        bucket: 

        encrypt: true

        secure: true

        v4auth: true

        chunksize: 5242880

        rootdirectory: /s3/object/name/prefix

    maintenance:

        uploadpurging:

            enabled: true

            age: 168h

            interval: 24h

            dryrun: false

        readonly:

            enabled: false

redis:

    addr: redis-host:6379

    password: asecret

    dialtimeout: 10ms

    readtimeout: 10ms

    writetimeout: 10ms

    pool:

        maxidle: 16

        maxactive: 64

        idletimeout: 300s

[code - typo] Page 220



The name of the file shown should be scalable-config.yml as in previous examples. This example also requires the addition of the newer uploadpurging attributes. The mainenance section of the file should be as follows:

    maintenance:

        uploadpurging:

            enabled: true

            age: 168h

            interval: 24h

            dryrun: false

        readonly:

            enabled: false

[text - typo] Page 240



In the second paragraph the reader is instructed to, "Open ./coffee/api/api.py" this is not the correct location of the file. The correct file location is at, "./coffee/app/api.py."

[text - typo] Page 262



The refere nce to "flock.json" in the first sentence of the third paragraph should be "flock.yml."

[code - typo] Page 270



The git clone command uses the ssh protocol instead of https. The command should read as follows:

git clone https://github.com/dockerinact ... i.git

import jieba.posseg as pseg

seg_list = pseg.cut("我爱北京天安门")

for word,flag in seg_list:

    print(word)

    print(flag)

---------------------------------------------------------------------------

TypeError                                 Traceback (most recent call last)

<ipython-input-5-f105f6980f88> in <module>()

      1 import jieba.posseg as pseg

      2 seg_list = pseg.cut("我爱北京天安门")

----> 3 for word,flag in seg_list:

      4     print(word)

      5     print(flag)



TypeError: cannot unpack non-iterable pair object

原因是新版本中seg_list是一个生成器，所以只能 for win seg_list

然后从word中解包出来

print(w.word)

print(w.flag)

这样问题就解决了。

from lxml.html import etree

from lxml.html import fromstring, HtmlElement



test_html = '''<p><span>hello</span><span>world</span></p>'''

test_element = fromstring(test_html)

etree.strip_tags(test_element,'span') # 清除span标签

etree.tostring(test_element)

url = 'http://xyhz.huizhou.gov.cn/static/js/common/jigsaw/images/{}.jpg'

headers={'User-Agent':'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'}

async def getPage(num):



    async with aiohttp.ClientSession() as session:

        async with session.get(url.format(num),headers=headers) as resp:

            if resp.status==200:

                f= await aiofiles.open('{}.jpg'.format(num),mode='wb')

                await f.write(await resp.read())

                await f.close()



loop = asyncio.get_event_loop()

tasks = [getPage(i) for i in range(5)]

loop.run_until_complete(asyncio.wait(tasks))

import time



LOOP = 2*10**8



def add(x,y):

    return x+y



def cpu_pressure(loop):

    

    for i in range(loop):

        result = add(i,i+1)





if __name__ == '__main__':

    start = time.time()

    cpu_pressure(LOOP)

    print(f'time used {time.time()-start}s')

class Command(ScrapyCommand):



    requires_project = True



    def syntax(self):

        return '[options]'



    def short_desc(self):

        return 'Runs all of the spiders - My Defined'



    def run(self,args,opts):

        print('==================')

        print(type(self.crawler_process))

        spider_list = self.crawler_process.spiders.list() # 找到爬虫类



        for name in spider_list:

            print('=================')

            print(name)

            self.crawler_process.crawl(name,**opts.__dict__)



        self.crawler_process.start()

      def __init__(self, settings=None):

        if isinstance(settings, dict) or settings is None:

            settings = Settings(settings)

        self.settings = settings

        self.spider_loader = _get_spider_loader(settings) # 构造爬虫

        self._crawlers = set()

        self._active = set()

        self.bootstrap_failed = False

def _get_spider_loader(settings):



    cls_path = settings.get('SPIDER_LOADER_CLASS')

    

    # settings文件没有定义SPIDER_LOADER_CLASS，所以这里获取到的是系统的默认配置文件，

    # 默认配置文件在接下来的代码块A

    # SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader'    

    

    loader_cls = load_object(cls_path) 

    # 这个函数就是根据路径转为类对象，也就是上面crapy.spiderloader.SpiderLoader 这个

    # 字符串变成一个类对象

    # 具体的load_object 对象代码见下面代码块B



    return loader_cls.from_settings(settings.frozencopy())

# 代码块A

#......省略若干

SCHEDULER = 'scrapy.core.scheduler.Scheduler'

SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleLifoDiskQueue'

SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.LifoMemoryQueue'

SCHEDULER_PRIORITY_QUEUE = 'scrapy.pqueues.ScrapyPriorityQueue'



SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader' 就是这个值

SPIDER_LOADER_WARN_ONLY = False



SPIDER_MIDDLEWARES = {}

# 代码块B 为了方便，我把异常处理的去除

from importlib import import_module #导入第三方库



def load_object(path):

    dot = path.rindex('.') 

    module, name = path[:dot], path[dot+1:]

    # 上面把路径分为基本路径+模块名

    

    mod = import_module(module)

    obj = getattr(mod, name)

    # 获取模块里面那个值

    

    return obj

In [33]: mod = import_module(module)                                                                                                                                             



In [34]: mod                                                                                                                                                                     

Out[34]: <module 'scrapy.spiderloader' from '/home/xda/anaconda3/lib/python3.7/site-packages/scrapy/spiderloader.py'>



In [35]: getattr(mod,name)                                                                                                                                                       

Out[35]: scrapy.spiderloader.SpiderLoader



In [36]: obj = getattr(mod,name)                                                                                                                                                 



In [37]: obj                                                                                                                                                                     

Out[37]: scrapy.spiderloader.SpiderLoader



In [38]: type(obj)                                                                                                                                                               

Out[38]: type

    def from_settings(cls, settings):

        return cls(settings)

class SpiderLoader(object):

    """

    SpiderLoader is a class which locates and loads spiders

    in a Scrapy project.

    """

    def __init__(self, settings):

        self.spider_modules = settings.getlist('SPIDER_MODULES') 

        # 获得settting中的模块名字，创建scrapy的时候就默认帮你生成了

        # 你可以看看你的settings文件里面的内容就可以找到这个值，是一个list

        

        self.warn_only = settings.getbool('SPIDER_LOADER_WARN_ONLY')

        self._spiders = {}

        self._found = defaultdict(list)

        self._load_all_spiders() # 加载所有爬虫

def _load_all_spiders(self):

        for name in self.spider_modules:            



                for module in walk_modules(name): # 这个遍历文件夹里面的文件，然后再转化为类对象，

                    # 保存到字典：self._spiders = {}       

                    self._load_spiders(module) # 模块变成spider



        self._check_name_duplicates() # 去重，如果名字一样就异常

def iter_spider_classes(module):

    from scrapy.spiders import Spider



    for obj in six.itervalues(vars(module)): # 找到模块里面的变量，然后迭代出来

        if inspect.isclass(obj) and \

           issubclass(obj, Spider) and \

           obj.__module__ == module.__name__ and \

           getattr(obj, 'name', None): # 有name属性，继承于Spider

           yield obj