TUTORIAL Google Contacts to Asterisk Phonebook

Discussion in 'PIAF 3 Add-Ons' started by raphou, Aug 4, 2011.

  1. raphou

    raphou Member

    Joined:
    Nov 8, 2008
    Messages:
    72
    Likes Received:
    1
    Hi guys,

    I've found a script to dump all your Google contacts to your asterisk phonebook. I have tweeked it a little bit to allow non-US users to use it.
    I strongly advice you to use the 2-stage login with gmail, as the password is stored in clear (same for the Gvoice calling with freepbx).

    1. Set-up the 2-step login
    2. Make sure Python is installed and ElementTree is installed (For Help)
    3. Download and install (./setup.py after unzip) the Gdata-python libraries
    4. Download and Edit John Baab's script. using your informaton
    Code:
    #!/usr/bin/python
    # googlecontacts.py v0.1
    # By: John Baab
    # Email: [email protected]
    # Purpose: syncs contacts from google to asterisk server
    # Requirements: python, gdata python client, asterisk
    #
    # License:
    #
    # This Package is free software; you can redistribute it and/or
    # modify it under the terms of the GNU General Public
    # License as published by the Free Software Foundation; either
    # version 3 of the License, or (at your option) any later version.
    #
    # This package is distributed in the hope that it will be useful,
    # but WITHOUT ANY WARRANTY; without even the implied warranty of
    # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    # General Public License for more details.
    #
    # You should have received a copy of the GNU General Public
    # License along with this package; if not, write to the Free Software
    # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
    #
    # On Debian & Ubuntu systems, a complete copy of the GPL can be found under
    # /usr/share/common-licenses/GPL-3, or (at your option) any later version
    
    import atom,re,sys,os
    import gdata.contacts
    import gdata.contacts.service
    
    def main():
        # Change this if you aren't in the US.  If you have more than one country code in your contacts,
        # then use an empty string and make sure that each number has a country code.
        country_code = "1"
    
        gd_client = gdata.contacts.service.ContactsService()
        gd_client.email = "[email protected]"
        gd_client.password = "and your pass"
        gd_client.source = 'gcontact2ast'
        gd_client.ProgrammaticLogin()
        query = gdata.contacts.service.ContactsQuery()
        query.max_results = 1000
        feed = gd_client.GetContactsFeed(query.ToUri())
    
        # delete all of our contacts before we refetch them, this will allow deletions
        os.system("asterisk -rx \'database deltree cidname\'")
    
        # for each phone number in the contacts
        for i, entry in enumerate(feed.entry):
            for phone in entry.phone_number:
                # Strip out any non numeric characters
                phone.text = re.sub('\D', '', phone.text)
    
                # Remove leading digit if it exists, we will add this again later for all numbers
                # Only if a country code is defined.
                if country_code != "":
                    phone.text = re.sub('^\+?%s' % country_code, '', phone.text)
    
                # Insert the number into the cidname database, reinsert the country code if defined.
                os.system("asterisk -rx \'database put cidname +%s%s \"%s\"\'" % (country_code,phone.text,entry.title.text))
    
    
    if __name__ == "__main__":
            main()
    
    
    5. Chmod+x it
    6. Execute it and voilà!

    You can of course make it executable every week (it's what I did to have updated information)

    Here is my modified version to allow more International numbers (all my contacts are store like this +countrycode areacode w number, eg: +3228888888)

    Code:
    #!/usr/bin/python
    # googlecontacts.py v0.1
    # By: John Baab
    # Email: [email protected]
    # MODIFIED by : Raphael
    # Purpose: syncs contacts from google to asterisk server
    # Requirements: python, gdata python client, asterisk
    #
    # License:
    #
    # This Package is free software; you can redistribute it and/or
    # modify it under the terms of the GNU General Public
    # License as published by the Free Software Foundation; either
    # version 3 of the License, or (at your option) any later version.
    #
    # This package is distributed in the hope that it will be useful,
    # but WITHOUT ANY WARRANTY; without even the implied warranty of
    # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    # General Public License for more details.
    #
    # You should have received a copy of the GNU General Public
    # License along with this package; if not, write to the Free Software
    # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
    #
    # On Debian & Ubuntu systems, a complete copy of the GPL can be found under
    # /usr/share/common-licenses/GPL-3, or (at your option) any later version
    
    import atom,re,sys,os
    import gdata.contacts
    import gdata.contacts.service
    
    def main():
        
        gd_client = gdata.contacts.service.ContactsService()
        gd_client.email = "[email protected]"
        gd_client.password = "PASSWORD"
        gd_client.source = 'gcontact2ast'
        gd_client.ProgrammaticLogin()
        query = gdata.contacts.service.ContactsQuery()
        query.max_results = 1000
        feed = gd_client.GetContactsFeed(query.ToUri())
    
        # delete all of our contacts before we refetch them, this will allow deletions
        os.system("asterisk -rx \'database deltree cidname\'")
    
        # for each phone number in the contacts
        for i, entry in enumerate(feed.entry):
            for phone in entry.phone_number:
                # Strip out any non numeric characters
                phone.text = re.sub('\D', '', phone.text)
    	     
                # Insert the number into the cidname database, reinsert the 00 (international)code.
                
                os.system("asterisk -rx \'database put cidname 00%s \"%s\"\'" % (phone.text,entry.title.text))
    	     
    if __name__ == "__main__":
            main()
    
    
    There are other scripts around that connect each time to Google to check the Caller ID, but I prefer this solution as it's much faster and cleaner.

    Cheers!
     
    wardmundy likes this.
  2. kdaffef02

    kdaffef02 Member

    Joined:
    Apr 29, 2011
    Messages:
    107
    Likes Received:
    6
    syntax error after gdata-2.0.15 installation ??

    Hi, Excuse me for my littre experience with Piaf.

    I'm using Piaf 1.7.5.6 with 2.8.1.4 freepbex version on a local hardware (asterisk 1.6.2.20)
    During the installation, i get the error message above. Should I continue the installation ??


    Thank you for help.

    Kamel

    ==========================================
    [email protected]:/usr/bin/gdata-2.0.15 $ ./setup.py install
    /usr/lib/python2.4/distutils/dist.py:236: UserWarning: Unknown distribution option: 'install_requires'
    warnings.warn(msg)
    running install
    running build
    running build_py
    running install_lib
    byte-compiling /usr/lib/python2.4/site-packages/gdata/apps/migration/service.py to service.pyc
    File "/usr/lib/python2.4/site-packages/gdata/apps/migration/service.py", line 198
    0 if num_entries % threads_per_batch == 0 else 1)
    ^
    SyntaxError: invalid syntax
     
  3. trupsalms

    trupsalms Member

    Joined:
    May 13, 2011
    Messages:
    314
    Likes Received:
    4
    can you please provide a little more instruction

    1. Set-up the 2-step login: self explanatory

    2. Make sure Python is installed and ElementTree is installed (For Help): details on how to compile and configure this set up

    3. Download and install (./setup.py after unzip) the Gdata-python libraries : details on how to compile and configure this set up

    4. Download and Edit John Baab's script. using your informaton: details on how to compile and configure this set up
     
  4. GDProjects

    GDProjects New Member

    Joined:
    Jun 24, 2013
    Messages:
    1
    Likes Received:
    0
    Hi

    Can anyone help with this, I need a bit more advice on finishing this install too.

    I've installed Python and created my custom version of the .py file (enabling 2 step authentication etc).

    When I run this I get the following:

    I've got the following errors:

    Traceback (most recent call last):
    File "./gclookup.py", line 30, in ?
    import gdata.contacts.service
    File "/usr/lib/python2.4/site-packages/gdata/contacts/service.py", line 32, in ?
    import gdata.service
    File "/usr/lib/python2.4/site-packages/gdata/service.py", line 81, in ?
    import gdata.auth
    File "/usr/lib/python2.4/site-packages/gdata/auth.py", line 39, in ?
    import gdata.gauth
    File "/usr/lib/python2.4/site-packages/gdata/gauth.py", line 66, in ?
    import json as simplejson
    Help!!
     
  5. wardmundy

    wardmundy Nerd Uno

    Joined:
    Oct 12, 2007
    Messages:
    14,432
    Likes Received:
    2,450
    [​IMG]

    Not many steps to import Google Contacts into Asterisk Phonebook with PIAF-Purple or PIAF-Green or Incredible PBX for Raspberry Pi...

    Code:
    cd /root
    mkdir Google
    cd Google
    wget https://gdata-python-client.googlecode.com/files/gdata-2.0.18.tar.gz
    tar zxvf gdata*
    cd gdata*
    chmod +x setup.py
    ./setup.py install
    nano -w googlecontacts.py
    
    Next, copy and paste desired raphou code above and insert your Google credentials into googlecontacts.py. If you're using 2-step authentication with your Google account, remember to generate and use an application-specific password. Regular PW won't work!

    For folks in the U.S. that only want 10-digit CallerID numbers, you need to make 2 additional changes to the code.
    First, change line 34 to: country_code = ""
    Second, change line 60 to: os.system("asterisk -rx \'database put cidname %s%s \"%s\"\'" % (country_code,phone.text,entry.title.text))

    NOTE: If you have more than 1,000 phone numbers in your Google Contacts, be sure to modify line 42 accordingly.

    Now save the script and make it executable: chmod +x googlecontacts.py
    Then run the script: ./googlecontacts.py

    Check your entries: asterisk -rx "database show cidname"


    Modify CallerID Superfecta Lookup Sources in FreePBX to include Asterisk Phonebook:

    [​IMG]

    For the perfect phone interface to your new Asterisk Phonebook, see the Yealink T46G thread.

    [​IMG]

    You can automate the import process to keep your Google contacts current in Asterisk by adding the script to /etc/crontab and running it each day:

    Code:
    9 0 * * * root /root/Google/gdata-2.0.18/googlecontacts.py >/dev/null 2>&1
     
    The Deacon and matt91 like this.
  6. Brian Simmons

    Brian Simmons Active Member

    Joined:
    May 22, 2013
    Messages:
    165
    Likes Received:
    34
    Is it possible to use this to import contacts from two different Google accounts? My wife and my accounts for example. Would it duplicate entries if we both had the same people/numbers in our contacts?
     
  7. accountingnerd

    accountingnerd New Member

    Joined:
    Mar 17, 2011
    Messages:
    6
    Likes Received:
    5
    Brian Simmons: You could import contacts from two different google accounts by making a copy of this script, and commenting out/deleting the section that deletes all the data in the second script, and then running the second script after running the first. That's the quick and dirty method. Doing that would definitely duplicate entries if there were duplicates between the two accounts. There is no duplicate checking in this script.
     
  8. Tyler

    Tyler New Member

    Joined:
    May 22, 2013
    Messages:
    14
    Likes Received:
    0
    Should this work for Google enterprise users? I get a Request Error (status: 401) when running the script. I have two-stage authentication setup and verified that my app-specific passwords match accordingly.


    This is what is returned:
    File "./googlecontacts.py", line 64, in <module>
    main()
    File "./googlecontacts.py", line 43, in main
    feed = gd_client.GetContactsFeed(query.ToUri())
    File "/usr/lib/python2.6/site-packages/gdata/contacts/service.py", line 104, in GetContactsFeed
    return self.Get(uri, converter=gdata.contacts.ContactsFeedFromString)
    File "/usr/lib/python2.6/site-packages/gdata/service.py", line 1108, in Get
    'reason': server_response.reason, 'body': result_body}
    gdata.service.RequestError: {'status': 401, 'body': REDACTED
     
  9. wardmundy

    wardmundy Nerd Uno

    Joined:
    Oct 12, 2007
    Messages:
    14,432
    Likes Received:
    2,450
  10. Tyler

    Tyler New Member

    Joined:
    May 22, 2013
    Messages:
    14
    Likes Received:
    0
    Thanks Ward. I did just try running the contacts example script which worked fine. Not sure what is going wrong on my end. I'll do little more head scratching and a bit more research to see if I can get to the bottom of it. I welcome any suggestion if anyone has any other ideas. The error being so cryptic is proving to be the biggest hurdle the moment.
     
  11. matt91

    matt91 Member

    Joined:
    Jul 29, 2010
    Messages:
    58
    Likes Received:
    5
    I ran the US script that Ward has posted on NerdVittles today, but i'm getting this error:
    Code:
    [email protected]:~/Google/gdata-2.0.18 $ ./googlecontacts.py
      File "./googlecontacts.py", line 34
        country_code = ""
                   ^
    IndentationError: expected an indented block
    [email protected]:~/Google/gdata-2.0.18 $
    
    The only changes I did were to add my gmail user/pass to the script.
    any thoughts?
    thanks all
    Matt
     
  12. matt91

    matt91 Member

    Joined:
    Jul 29, 2010
    Messages:
    58
    Likes Received:
    5


    Answering my own question, you have to insert 4-space indents just as shown in the OP of this thread. Simply copying the text from nerdvittles, as I did, leaves it unformatted.
     
  13. wardmundy

    wardmundy Nerd Uno

    Joined:
    Oct 12, 2007
    Messages:
    14,432
    Likes Received:
    2,450
    Looks like WordPress lost the required indents with certain browsers. Try the following:
    Code:
    cd /root/Google/gdata-2.0.18
    rm googlecontacts.py
    wget http://pbxinaflash.com/googlecontacts.py
    nano -w googlecontacts.py
    Then make the changes outlined in the article and save the modified file. We've adjusted the Nerd Vittles article to resolve this. Thanks for the heads up.
     
  14. Jay Schulman

    Jay Schulman New Member

    Joined:
    Jul 8, 2013
    Messages:
    1
    Likes Received:
    3
    All,

    For those getting a "gdata.service.RequestError: {'status': 401, 'body':...." error with a Google Apps account, it's because Google is forcing you to use the new Google Contacts API v3.

    I've updated the script to support the new API and Google apps users should now be functional. You need to update the script with:

    email = '[email protected]'
    password = 'password'
    domain = 'domain.com'

    domain.com is your google apps hosted domain.

    I've set the maximum results to 2000. You'll need to increase that should you have more.

    Thanks.

    Code:
    #!/usr/bin/python
    # googlecontacts.py v0.1apps
    # Original By: John Baab
    # Email: [email protected]
    # Updated By: Jay Schulman
    # Email: [email protected]
    # Purpose: syncs contacts from google to asterisk server
    # Updates: Updating for Google API v3 with support for Google Apps
    # Requirements: python, gdata python client, asterisk
    #
    # License:
    #
    # This Package is free software; you can redistribute it and/or
    # modify it under the terms of the GNU General Public
    # License as published by the Free Software Foundation; either
    # version 3 of the License, or (at your option) any later version.
    #
    # This package is distributed in the hope that it will be useful,
    # but WITHOUT ANY WARRANTY; without even the implied warranty of
    # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    # General Public License for more details.
    #
    # You should have received a copy of the GNU General Public
    # License along with this package; if not, write to the Free Software
    # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
    #
    # On Debian & Ubuntu systems, a complete copy of the GPL can be found under
    # /usr/share/common-licenses/GPL-3, or (at your option) any later version
     
    import atom,re,sys,os
    import gdata.data
    import gdata.auth
    import gdata.contacts
    import gdata.contacts.client
    import gdata.contacts.data
     
    email = '[email protected]'
    password = 'password'
    domain = 'domain.com'
     
    def main():
        # Change this if you aren't in the US.  If you have more than one country code in your contacts,
        # then use an empty string and make sure that each number has a country code.
        country_code = ""
     
        gd_client = gdata.contacts.client.ContactsClient(domain=domain)
        gd_client.ClientLogin(email, password, 'google2asterisk')
        qry = gdata.contacts.client.ContactsQuery(max_results=2000)
        feed = gd_client.GetContacts(query=qry)
     
        # delete all of our contacts before we refetch them, this will allow deletions
        os.system("asterisk -rx \'database deltree cidname\'")
     
        # for each phone number in the contacts
        for i, entry in enumerate(feed.entry):
            for phone in entry.phone_number:
                # Strip out any non numeric characters and convert to UTF-8
                phone.text = re.sub('[^0-9]', '', phone.text)
            phone.text = unicode(phone.text, 'utf8')
     
                # Remove leading digit if it exists, we will add this again later for all numbers
                # Only if a country code is defined.
                if country_code != "":
                    phone.text = re.compile('^\+?%s' % country_code, '', phone.text)
     
                os.system("asterisk -rx \'database put cidname %s%s \"%s\"\'" % (country_code,phone.text,entry.title.text))
     
     
    if __name__ == "__main__":
            main()
    
     
    Rrrr, Huckda and wardmundy like this.
  15. Rrrr

    Rrrr Guru

    Joined:
    May 28, 2009
    Messages:
    319
    Likes Received:
    22
    Thanks for the elegant solution!
    However, I am running into trouble with my google apps domain.

    I set max_results=5 and get 0 results found.

    Here is what I did
    1. Ran install instructions from here
    2. Copied Jay's script above in googlecontacts.py and set max_results=5
    3. (Note: needed fixing two small errors after copying Jay's modified script above : 1. align phone text.entry with spaces, 2. remove bad character line 66)
    4. Verified first 5 contacts have country code +XX in google contacts and therefore I left country_code = "" untouched

    Q1: Where should I look to fix this?

    Then I tried setting country code, but received an error:
    It seems that the number of arguments given to re.compile is wrong.
    Q2: I noticed however, that the downloaded script from the original install has a different line
    but I do not know if this could be a fix?
    Code:
                if country_code != "":
                    # phone.text = re.compile('^\+?%s' % country_code, '', phone.text)
                    phone.text =      re.sub('^\+?%s' % country_code, '', phone.text)
    

    Incredible PBX 3.6 for Raspberry Pi
    Asterisk 1.8.19.1 Freepbx 2.10.10
     
  16. Rrrr

    Rrrr Guru

    Joined:
    May 28, 2009
    Messages:
    319
    Likes Received:
    22
    Solved: I installed Incredible PBX 3.11 for Raspberry Pi and now the script is updating the database and really creating entries.

    However the second part (see Q2) is still not working: the re.compile statement is wrong.
    The re.sub statement is correctly adding the country code in front of the numbers (be it that it blindly assumes all the numbers are in local format): Solved
     
  17. Rrrr

    Rrrr Guru

    Joined:
    May 28, 2009
    Messages:
    319
    Likes Received:
    22
    ...:oops:

    I was smiling too quickly, the app chokes on some input: names containing the letter ö or ü or é....
    Do you have an idea how to change the UTF8 conversion into something else, perhaps ascii?
     
  18. Rrrr

    Rrrr Guru

    Joined:
    May 28, 2009
    Messages:
    319
    Likes Received:
    22
    I modified the script to sync selected Google contacts to the Asteridex SQL database.
    It took me a while, but it is working (5000+ contacts) and its fast.

    Still needs much cleaning up...I have not programmed for 20 years, let alone OOP, but it was fun :). So in case you are interested, let me know, but you were warned.

    Modifications:
    # 1. Sync selected contact group from Google to MYSQL database on asterisk server (Asteridex format)
    # 2. Generate dial codes for a different contact group, if given
    # (eg. for CallWho and Asteridex robo dialer)
    # 3. Unify international prefixes to + or 00 or 011 (currently, Superfecta seems not handle + well)
    # 4. Allow names that contain characters with accents in the data without crashing
    # 5. Allow dynamic CLI input for the contact groups
    #
    # Tip: In order to benefit from CallerID lookup using the SQL database, use eg. Caller ID Superfecta
    # Remember to set Database as Data Source Name and complete its fields as below, including the
    # field containing SQL Query as follows: SELECT `name` FROM `user1` WHERE `out`= :thenumber
     
  19. Tyler

    Tyler New Member

    Joined:
    May 22, 2013
    Messages:
    14
    Likes Received:
    0
    Rrrr,
    I'd love to give it a try if you are okay with sharing. I think the modification you've made might solve some of my issues.
     
  20. accountingnerd

    accountingnerd New Member

    Joined:
    Mar 17, 2011
    Messages:
    6
    Likes Received:
    5
    Here's my version of the Contacts to Asteridex. I had posted this prior to the Great Crash, but I think it was one of the lost items. In addition to the standard requirements, you'll also need mysql-python (http://mysql-python.sourceforge.net/MySQLdb.html). Remember, if you run this, it will wipe out your existing Asteridex database, so as always, make and test your backup, use at your own risk, etc.

    Code:
    #!/usr/bin/python
    # googlecontacts.py v0.1 By: John Baab Email: [email protected]
    # Modified by: Aaron J. Clark, CPA.CITP Email: [email protected]
    # Purpose: syncs contacts from google to asteridex
    # Requirements: python, gdata python client, asterisk, MySQLdb
    #
    # License:
    #
    # This Package is free software; you can redistribute it and/or modify
    # it under the terms of the GNU General Public License as published by
    # the Free Software Foundation; either version 3 of the License, or (at
    # your option) any later version.
    #
    # This package is distributed in the hope that it will be useful, but
    # WITHOUT ANY WARRANTY; without even the implied warranty of
    # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    # General Public License for more details.
    #
    # You should have received a copy of the GNU General Public License
    # along with this package; if not, write to the Free Software
    # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301
    # USA
    #
    # On Debian & Ubuntu systems, a complete copy of the GPL can be found
    # under /usr/share/common-licenses/GPL-3, or (at your option) any later
    # version
    import atom,re,sys,os
    import gdata.contacts
    import gdata.contacts.service
    import MySQLdb
     
    def main():
     
        # Change this if you aren't in the US.  If you have more than one
        # country code in your contacts, then use an empty string and make
        # sure that each number has a country code.
        country_code = "1"
        gd_client = gdata.contacts.service.ContactsService()
        gd_client.email = "[email protected]" # Or use your google apps account
        gd_client.password = "password" # Highly recommend that you use 2-factor authentication
        gd_client.source = 'gcontact2ast'
        gd_client.ProgrammaticLogin()
        query = gdata.contacts.service.ContactsQuery()
        query.max_results = 4000
        feed = gd_client.GetContactsFeed(query.ToUri())
        # Create a connection to MySQL
        con = MySQLdb.connect(host='localhost', user='root', passwd='passw0rd', db='asteridex')
        cur = con.cursor()
        # delete all of our contacts before we refetch them, this will propogate deletions
        cur.execute("DELETE FROM user1")
        cur.close()
       
        # re-instantiate the cursor
        cur = con.cursor()
       
        # for each phone number in the contacts
        for i, entry in enumerate(feed.entry):
            for phone in entry.phone_number:
     
                # I don't want to bring in phone numbers of type "Other"
                if phone.rel != gdata.contacts.REL_OTHER:
       
                    # Strip out any non numeric characters
                    phone.text = re.sub('\D', '', phone.text)
                   
                    # Remove leading digit if it exists, we
                    # will add this again later for all
                    # numbers Only if a country code is
                    # defined.
                    if country_code != "":
                        phone.text = re.sub('^\+?%s' % country_code, '', phone.text)
                   
                    # Insert the number into the cidname
                    # database, reinsert the country code if
                    # defined.
                   
                    name = entry.title.text
                    if str(name).strip() != "None":
                        phonetype = phone.rel.split("#")[1]
                        name = name + " (" + phonetype + ")"
                        num = phone.text
                        if str(name).strip() != "None":
                            cur.execute("INSERT INTO user1(`name`, `out`) VALUES (%s,%s)",(name,num))
                            print str(name) + " [" + num + "] was inserted into the database"
                   
        cur.close()
        con.close()
     
    if __name__ == "__main__":
        main()
    
     
    leemason and wardmundy like this.