Implement URL and IM field types in CardDAV

Partially takes care of: #104

asynk/contact_cd.py

@@ -63,6 +63,8 @@ class CDContact(Contact):
     ABDATE     = 'X-ABDATE'
     ABLABEL    = 'X-ABLABEL'
     OMIT_YEAR  = 'X-APPLE-OMIT-YEAR'
+    IMPP       = 'IMPP'
+    URL        = 'URL'
 
     ## OUr own extensions
     X_NOTE     = 'X-ASYNK_NOTE'
@@ -173,6 +175,9 @@ def init_props_from_vco (self, vco):
         self._snarf_names_gender_from_vco(vco)
         self._snarf_emails_from_vco(vco)
         self._snarf_phones_from_vco(vco)
+        self._snarf_ims_from_vco(vco)
+        self._snarf_websites_from_vco(vco)
+        self._snarf_postal_from_vco(vco)
         self._snarf_org_details_from_vco(vco)
         self._snarf_dates_from_vco(vco)
         self._snarf_sync_tags_from_vco(vco)
@@ -188,6 +193,9 @@ def init_vco_from_props (self):
         self._add_prodid_to_vco(vco)
         self._add_names_gender_to_vco(vco)
         self._add_emails_to_vco(vco)
+        self._add_ims_to_vco(vco)
+        self._add_websites_to_vco(vco)
+        self._add_postal_to_vco(vco)
         self._add_phones_to_vco(vco)
         self._add_org_details_to_vco(vco)
         self._add_dates_to_vco(vco)
@@ -339,6 +347,68 @@ def _snarf_phones_from_vco (self, vco):
                         self.add_phone_other(('Other', phone.value))
 
 
+    def _snarf_ims_from_vco (self, vco):
+        if not hasattr(vco, 'impp'):
+            return
+
+        ims = vco.contents['impp']
+        for im in ims:
+            im_types = im.params['TYPE'] if 'TYPE' in im.params else None
+
+            ## ASynK currently does not support custom labels for IMs in
+            ## CardDAV, which could be a potential problem.
+
+            label = im_types[0].lower() if len(im_types) > 0 else 'home'
+            if im_types:
+                im_types = [x.lower() for x in im_types]
+            else:
+                im_types = ['home']
+
+            ## Not exactly seen cases when we have more than one type in a
+            ## IMPP. But you never know
+            if len(im_types) > 1:
+                logging.debug('Contact %s has IMPP with more than one type: %s',
+                              self.get_name(), vco.contents['impp'])
+
+            for im_type in im_types:
+                if 'pref' == im_types:
+                    self.set_im_prim(im.value)
+
+                else:
+                    self.add_im(im_type, im.value)
+
+    def _snarf_websites_from_vco (self, vco):
+        if not hasattr(vco, 'url'):
+            return
+
+        urls = vco.contents['url']
+        for url in urls:
+            url_types = url.params['TYPE'] if 'TYPE' in url.params else None
+
+            if url_types:
+                url_types = [x.lower() for x in url_types]
+            else:
+                logging.debug('Contact %s has URL with no type. Assuming home',
+                              self.get_name())
+                url_types = ['home']
+
+            ## Note that the same URL can be stored as a prim AND a
+            ## home/work URL
+            if 'pref' in url_types:
+                self.set_web_prim(url.value)
+            if 'home' in url_types:
+                self.add_web_home(url.value)
+            if 'work' in url_types:
+                self.add_web_work(url.value)
+            else:
+                logging.debug('Sorry friend; you are going to lose some data ' +
+                              'on the www url for this unsupported type (%s) ' +
+                              'for contact: %s', url_types, self.get_name())
+
+    def _snarf_postal_from_vco (self, vco):
+        ## FIXME: To be implemented.
+        pass
+
     def _snarf_org_details_from_vco (self, vco):
         if hasattr(vco, l(self.TITLE)):
             self.set_title(getattr(vco, l(self.TITLE)).value)
@@ -506,6 +576,43 @@ def _add_emails_to_vco (self, vco):
         self._add_emails_to_vco_helper(vco, self.get_email_work, 'WORK')
         self._add_emails_to_vco_helper(vco, self.get_email_other, '')
 
+    def _add_ims_to_vco (self, vco):
+        im_prim = self.get_im_prim()
+        for label, addr in self.get_im().iteritems():
+            im = vco.add(l(self.IMPP))
+            im.value = addr
+            if im_prim == addr:
+                im.params.update({'TYPE' : [label, 'pref']})
+            else:
+                im.params.update({'TYPE' : [label]})
+
+    def _add_websites_to_vco (self, vco):
+        pref = self.get_web_prim()
+        home = self.get_web_home()
+        work = self.get_web_work()
+
+        if home and len(home) > 0:
+            for site in home:
+                url = vco.add(l(self.URL))
+                url.value = site
+                if pref == site:
+                    url.params.update({'TYPE' : ['home', 'pref']})
+                else:
+                    url.params.update({'TYPE' : ['home']})
+
+        if work and len(work) > 0:
+            for site in work:
+                url = vco.add(l(self.URL))
+                url.value = site
+                if pref == site:
+                    url.params.update({'TYPE' : ['work', 'pref']})
+                else:
+                    url.params.update({'TYPE' : ['work']})
+
+    def _add_postal_to_vco (self, vco):
+        ## FIXME: To be implemented.
+        pass
+
     def _add_phones_helper (self, vco, elem, pref, types, value):
         if not value:
             return

test/README.org

@@ -1 +1,120 @@
-This directory contains random test and exploration code that can be used to activate and test specific areas of the code base in isolation. In particular the "gold/" directory is meant to be the full unit test suite that every change should pass before commit. This directory, otoh, is to do some probing (like print a given outlook entry, or clear specific flags, or some such one time operation in a debug cycle).
+This directory contains random test and exploration code that can be
+used to activate and test specific areas of the code base in
+isolation. In particular the "gold/" directory is meant to be the full
+unit test suite that every change should pass before commit. This
+directory, otoh, is to do some probing (like print a given outlook
+entry, or clear specific flags, or some such one time operation in a
+debug cycle).
+
+* Misc bits about testing
+
+Some random bits of information relevant for testing. This section is
+not meant to be intelligble to anyone other than the author (skarra@)
+
+** CardDav testing
+
+*** Setup Baikal on localhost
+
+At various points in the past I have used CardDav servers from Baikal
+and Apple "Calendar Server".
+
+On upgrading to macOS Mojave stuff stopped working and I had to redo a
+bunch of things in Apr 2019:
+
+- Set up Apache, like so:
+  https://coolestguidesontheplanet.com/install-apache-mysql-php-on-macos-mojave-10-14/
+  (and a bunch of other stuff to get the basic Apache config in
+  order).
+
+- Set up Baikal: http://sabre.io/baikal/install/ Ran into this error:
+  "Baïkal needs to have write permissions in the Specific/ folder."
+  Had to make that directory world writable. Oh well.
+
+  Eventually http://localhost/baikal/html/admin/install worked well,
+  and was able to complete some basic config and baikal installation
+  was completed
+
+  Created a new test account with following credentials:
+  - username: unittest1 and unittest2
+  - email: [email protected]
+  - Password: <saved to Chrome profile> for URL localhost/baikal/html
+
+  This is super useful information for later reference:
+  http://sabre.io/dav/building-a-carddav-client/
+
+- Finally set up Baikal to be on a separate virtualhost with the
+  following config:
+
+<VirtualHost *:80>
+    DocumentRoot /Library/WebServer/Documents/baikal/html
+    ServerName baikal
+
+    RewriteEngine On
+    RewriteRule /.well-known/carddav /dav.php [R,L]
+    RewriteRule /.well-known/caldav /dav.php [R,L]
+
+    <Directory "/Library/WebServer/Documents/baikal/html">
+        Options None
+	Options +FollowSymlinks
+	AllowOverride All
+
+	# Confiugration for apache-2.2:
+	Order allow,deny
+	Allow from all
+
+	# Confiugration for apache-2.4:
+	Require all granted
+    </Directory>
+</VirtualHost>
+
+  Note that the main localhost stuff is as follows:
+
+<VirtualHost *:80>
+    DocumentRoot "/Library/WebServer/Documents"
+    ServerName localhost
+    <Directory "/Library/WebServer/Documents">
+        Options FollowSymLinks Multiviews
+        MultiviewsMatch Any
+        AllowOverride None
+        Require all granted
+    </Directory>
+</VirtualHost>
+
+***  Testing - MacOS Mojave Contacts does not work
+
+Also tried using caldavclientlibrary to inspect the carddav server
+
+- One of the main problemsin testing carddav automatically is to
+  effect changes on the server via "other means" - either
+  programmatically or manually through another interface. Setting up
+  Apple Contacts on Mac to read the carddav server has been a pain.
+
+  2019-04-27: The following config does not work:
+  - CardDAV, Manual
+  - username:password (same as what I used for ASynK)
+  - Server Address: http://localhost/baikal/html/card.php
+  - Server Path: /addressbooks/unittest1/default/
+
+  Tried a bunch of variations on the Server path to
+  /addressbooks/default/ etc but nothing works
+
+  I tried enabling some network logging in OSX, like so:
+
+     $ defaults write com.apple.AddressBook.CardDAVPlugin EnableDebug -bool YES
+     $ defaults write com.apple.AddressBook.CardDAVPlugin LogConnectionDetails -bool YES
+
+  And logs were supposed to show up using Console.app in
+  ~/Library/Logs; it does not work. Just WTF...
+
+*** Testing - BusyContacts works and displays contacts from Baikal
+
+Installed 30 day trial version of https://www.busymac.com/busycontacts/ and it just
+worked with the localhost version of Baikal.
+
+username: unittest1
+Password: <from chrome password manager>
+server: http://baikal/ - that's it.
+
+Testing flow was like so:
+
+- Clear all c

test/gold/README.org

@@ -1 +1,26 @@
-This directory contains unit tests that must pass before anything can be released. At this time the coverage is, er, rather poor. But the idea is this will evolve into a proper regression test suite that every commit / batch of commits should pass before being accepted to the master branch.
+This directory contains unit tests that must pass before anything can
+be released. At this time the coverage is, er, rather poor. But the
+idea is this will evolve into a proper regression test suite that
+every commit / batch of commits should pass before being accepted to
+the master branch.
+
+* 2019-04-22 Thoughts on CardDAV gold testing
+
+As of today there is no automated testing for CardDAV sync (or any of
+the others for that matter). Here are some thoughts on what it could be.
+
+CardDAV testing can be along following lines:
+
+Pre-requisites:
+  - CardDAV server on locahost
+  - test principal account
+
+Test 1
+- Clear all the contacts
+- Initialize ASynK with a sample BBDB file and write a bunch of
+  contacts to the CardDAV server
+- make a raw HTTP or other carddav query for contacts, and validate
+  number of contacts and contents.
+
+The last step might require us to use this project or something
+similar to: https://github.com/apple/ccs-caldavtester for automation.

test/gold/data/bb-cd/01.06-bbdb.v7.names-with-commas

@@ -0,0 +1,3 @@
+;; -*-coding: utf-8-emacs;-*-
+;;; file-format: 7
+["Debajeet, the Great" "Das"  nil nil ("HolidayIQ") (["Work" "+91 80 4115 3595"] ["Mobile" "+91 97381 65128"]) nil ("[email protected]") ((bbdb-id . "d476c03a-e5c8-11e1-8d79-3c07541b9945") (creation-date . "2012-08-10 09:47:58 +0000") (timestamp . "2012-08-10 09:50:06 +0000") (title . "Manager") (department . "B2B Marketing") (notes . "Does something in HolidayIQ :) Met him when went to their office in July 2012.")  (seen-in . "INBOX") ) nil]

test/gold/data/bb-cd/02.09-bbdb.v7.phones

@@ -0,0 +1,17 @@
+;; -*-coding: utf-8-emacs;-*-
+;;; file-format: 7
+;;
+;; One structurd phone number 
+;;["John" "Doe" nil nil nil (["Mobile" 0 0 0 60]) nil ("[email protected]") ((creation-date . "2013-12-06 14:33:49 +0000") (timestamp . "2013-12-20 17:56:49 +0000")) nil]
+;;
+;; One structured phone number with a dot notation
+["John" "Doe" nil nil nil (["Mobile" 0 0 0 60.8]) nil ("[email protected]") ((creation-date . "2013-12-06 14:33:49 +0000") (timestamp . "2013-12-20 17:56:49 +0000")) nil]
+;;
+;; One Unstructured European style number
+["John" "Doe" nil nil nil (["Mobile" "+91 44 2811 2640"]) nil ("[email protected]") ((creation-date . "2013-12-06 14:33:49 +0000") (timestamp . "2013-12-20 17:56:49 +0000")) nil]
+;;
+;; Two variants of structured number
+["John" "Doe" nil nil nil (["Mobile" 0 0 0 60.8] ["Mobile" 0 0 0 60]) nil ("[email protected]") ((creation-date . "2013-12-06 14:33:49 +0000") (timestamp . "2013-12-20 17:56:49 +0000")) nil]
+;;
+;; All variants in single entry
+["John" "Doe" nil nil nil (["Mobile" 0 0 0 60.8] ["Mobile" 0 0 0 60] ["Mobile" "+91 44 2811 2640"]) nil ("[email protected]") ((creation-date . "2013-12-06 14:33:49 +0000") (timestamp . "2013-12-20 17:56:49 +0000")) nil]

test/gold/data/bb-cd/state.test.json

@@ -0,0 +1,32 @@
+// -*- javascript -*-
+
+// This is the starter file for test scripts doing testing between
+// BBDB and CardDAV
+
+{
+    'file_version' : 4,
+    'default_profile' : null,
+
+    'profiles' : {
+	"testbbcd" : {
+	    "coll_1" : {
+		"dbid" : "bb",
+		"foid" : "default",
+		"stid" : "user_dir/01.06-bbdb.v7.names-with-commas"
+	    },
+
+            "coll_2" : {
+		"dbid" : "cd",
+		"foid" : "default"
+		"stid" : "http://localhost/baikal/html/card.php"
+            },
+
+            "conflict_resolve" : "bb",
+            "items" : {},
+          "last_sync_start" : "2000-01-20T13:25:20.16Z",
+          "last_sync_stop" : "2000-01-20T13:25:20.86Z",
+          "olgid" : null,
+          "sync_dir" : "SYNC2WAY"
+        },
+    }, // 'profiles'
+}

test/gold/data/cd/06-im.vcf

@@ -0,0 +1,13 @@
+BEGIN:VCARD
+VERSION:3.0
+PRODID:-//ASynK v2.2.0+//EN
+UID:29f46d0f-04f4-11e7-a1c4-9801a799b56f
+EMAIL;TYPE=INTERNET,pref,WORK:[email protected]
+FN:Ailis Brocato
+N:Brocato;Ailis;;;
+ORG:To-Me;
+REV:20170318T110955
+X-ASYNK-CREATED:20170309T181358Z
+X-ASYNK-SYNCTAG-OWNCLD91-BB:d21ae84c-a4e2-11e1-ae00-3c07541b9945
+IMPP;TYPE=HOME:home_im
+END:VCARD