Kezunlin's Blog

Live and Learn

0%

pymongo and mongoengine tutorial on ubuntu 16.04

Posted on Edited on In

pymongo and mongoengine tutorial on ubuntu 16.04

Guide

version

  • mongo 2.6.10
  • mongo gui: robo3t-1.3.1
  • pymongo 3.9.0
  • MongoEngine 0.18.2

install mongodb

1
sudo apt-get install -y mongodb

allow remote access

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
# By default, MongoDB is listening on 127.0.0.1:27017 only:

$ sudo netstat -tnlp
Proto Recv-Q Send-Q Local Address   Foreign Address State  PID/Program name
tcp        0      0 127.0.0.1:27017 0.0.0.0:*       LISTEN 1106/mongod


vim /etc/mongod.conf
#bind_ip = 127.0.0.1
bind_ip = 0.0.0.0

sudo service mongodb restart

$ sudo netstat -tnlp
Proto Recv-Q Send-Q Local Address   Foreign Address State  PID/Program name
tcp        0      0 0.0.0.0:27017   0.0.0.0:*       LISTEN 1107/mongod
tcp        0      0 0.0.0.0:5000    0.0.0.0:*       LISTEN 2364/python  
tcp        0      0 0.0.0.0:22      0.0.0.0:*       LISTEN 4365/sshd

mongo shell

1
2
3
4
5
mongo --host mongodb0.example.com --port 27017
mongo 127.0.0.1:27017/myfb

mongo --version
MongoDB shell version: 2.6.10

see mongo shell

show dbs and collections

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
> mongo 
# show all dbs
> show dbs

# display current db
>db
test (default database)

# switch or create db
> use mydb
> db.help()

> show collections
posts
system.indexes

> db.posts.help()

> db.posts.find()

drop db and collection

1
2
>db.dropDatabase()
>db.posts.drop()

backup copy db

1
>db.copyDatabase("mydb","backup_mydb")

rename field

see here

1
2
3
4
5
6
7
8
# The `false, true` in the method above are: { upsert:false, multi:true }. 
# You need the multi:true to update all your records.

> db.students.update({}, { $rename: { "oldname": "newname" } }, false, true )
WriteResult({ "nMatched" : 48130, "nUpserted" : 0, "nModified" : 48130 })

#In MongoDB 3.2 you can also use
>db.students.updateMany( {}, { $rename: { "oldname": "newname" } } )

add new field

see here

Same as the updating existing collection field, $set will add a new fields if the specified field does not exist.

1
2
3
4
5
6
# The `false, true` in the method above are: { upsert:false, multi:true }. 
# You need the multi:true to update all your records.


db.your_collection.update({},{ $set: {"new_field": "field_value"} },false,true)
WriteResult({ "nMatched" : 775471, "nUpserted" : 0, "nModified" : 775471 })

indexes

syntax

#db.collection_name.createIndex({field_name: 1 or -1})
db.index_basic.createIndex({ts_code:1})

db.index_basic.getIndexes()

db.index_basic.dropIndex({ts_code:1})
db.index_basic.dropIndexes() # can not drop default `_id` index

The value 1 is for ascending order and -1 is for descending order

example

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
> db.index_basic.createIndex({ts_code:1})
{
        "createdCollectionAutomatically" : false,
        "numIndexesBefore" : 1,
        "numIndexesAfter" : 2,
        "ok" : 1
}
> db.index_basic.getIndexes()
[
        {
                "v" : 1,
                "key" : {
                        "_id" : 1
                },
                "name" : "_id_",
                "ns" : "stock.index_basic"
        },
        {
                "v" : 1,
                "key" : {
                        "ts_code" : 1
                },
                "name" : "ts_code_1",
                "ns" : "stock.index_basic"
        }
]
> 
> 
> db.index_basic.dropIndex({ts_code:1})
{ "nIndexesWas" : 2, "ok" : 1 }
> db.index_basic.getIndexes()
[
        {
                "v" : 1,
                "key" : {
                        "_id" : 1
                },
                "name" : "_id_",
                "ns" : "stock.index_basic"
        }
]
> db.index_basic.dropIndexes()
{
        "nIndexesWas" : 1,
        "msg" : "non-_id indexes dropped for collection",
        "ok" : 1
}

remove field

1
2
3
4
5
6
7
8
# remove field from all documents
> db.stock_basic
> db.stock_basic.findOne()
> db.stock_basic.update({},{$unset: {list_date_obj:1}},{multi: true});
WriteResult({ "nMatched" : 3717, "nUpserted" : 0, "nModified" : 3717 })
> db.stock_basic.update({},{$unset: {_StockBasic__created:1}},{multi: true});
> db.stock_basic.update({},{$unset: {_StockBasic__updated:1}},{multi: true});
> db.stock_basic.findOne()

mongo command

dump and restore db

  • mongoexport / mongoimport
  • mongodump / mongorestore
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
# export db to outdb on node1
mongodump --db myfb --out outdb
zip outdb.zip -r outdb/
scp outdb.zip myfb:~/

# import db on node2
unzip outdb.zip
mongorestore outdb

# check db file size
du -h outdb.zip
742M 

du -h outdb
9.1G 

# check to see 
mongo
> show dbs;
myfb 3.1G

mongo gui

Robomongo offically changed it’s name and released two different products Studio 3T and Robo 3T. Old robomongo is now called Robo 3T. Studio 3T is for professionals.

wget https://download-test.robomongo.org/linux/robo3t-1.3.1-linux-x86_64-7419c406.tar.gz

vim .bashrc
export PATH=/home/kezunlin/program/robo3t/bin:$PATH

allow mongodb to access from remote.

vim /etc/mongodb.conf
#bind_ip = 127.0.0.1
bind_ip = 0.0.0.0

by default, mongodb only allow to access from local.

restart mongodb again

 > sudo service mongodb status
 mongodb.service - An object/document-oriented database
   Loaded: loaded (/lib/systemd/system/mongodb.service; enabled; vendor preset: enabled)
   Active: active (running) since 四 2019-09-26 16:11:03 CST; 7s ago
     Docs: man:mongod(1)
 Main PID: 4917 (mongod)
    Tasks: 10
   Memory: 3.0G
      CPU: 70ms
   CGroup: /system.slice/mongodb.service
           └─4917 /usr/bin/mongod --config /etc/mongodb.conf

9月 26 16:11:03 node17 systemd[1]: Started An object/document-oriented database.
9月 26 16:11:03 node17 mongod[4917]: warning: bind_ip of 0.0.0.0 is unnecessary; listens on all ips by default

access from remote now 

robo3t

python mongodb

pip install pymongo
pip install mongoengine

One library that provides a higher abstraction on top of PyMongo is MongoEngine. MongoEngine is an object document mapper (ODM), which is roughly equivalent to a SQL-based object relational mapper (ORM). The abstraction provided by MongoEngine is class-based, so all of the models you create are classes.

pymongo

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
def test_mongo_client():

    from pymongo import MongoClient
    client = MongoClient('localhost', 27017)

    db = client['mydb']

    posts = db.posts
    post_data = {
        'title': 'Python and MongoDB',
        'content': 'PyMongo is fun, you guys',
        'author': 'Scott'
    }
    result = posts.insert_one(post_data)
    print('One post: {0}'.format(result.inserted_id))


    post_1 = {
        'title': 'Python and MongoDB',
        'content': 'PyMongo is fun, you guys',
        'author': 'Scott'
    }
    post_2 = {
        'title': 'Virtual Environments',
        'content': 'Use virtual environments, you guys',
        'author': 'Scott'
    }
    post_3 = {
        'title': 'Learning Python',
        'content': 'Learn Python, it is easy',
        'author': 'Bill'
    }
    new_result = posts.insert_many([post_1, post_2, post_3])
    print('Multiple posts: {0}'.format(new_result.inserted_ids))

    # find one 
    bills_post = posts.find_one({'author': 'Bill'})
    print(bills_post)

    # fine many
    scotts_posts = posts.find({'author': 'Scott'})
    print(scotts_posts)

    for post in scotts_posts:
        print(post)

    client.close()

mongoengine

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
from mongoengine import *

def test_mongo_engine():
    # define collection 
    class Post(Document):
        title = StringField(required=True, max_length=200)
        content = StringField(required=True)
        author = StringField(required=True, max_length=50)
        published = DateTimeField(default=datetime.datetime.now)

    connect('mydb', 
        host='localhost', 
        port=27017, 
        alias="default" # must be `default`
    )
    # mongoengine.connection.MongoEngineConnectionError: You have not defined a default connection

    post_1 = Post(
        title='Sample Post',
        content='Some engaging content',
        author='Scott'
    )
    post_1.save()       # This will perform an insert
    print(post_1.title)
    print(post_1.id)

    post_1.title = 'A Better Post Title'
    post_1.save()       # This will perform an atomic edit on "title"
    print(post_1.title)
    print(post_1.id)

    disconnect(alias='default')

test_mongo_engine()

Reference

History

  • 20190926: created.