Project: titanium
Issue TIMOB-17472: Tooling: Analyze CLI API analytics data for top-used functionality
Our goal is to convert the API data from the CLI into useful information about which are the most used APIs High-level process: # Grab compressed JSON files from /appcelerator.analytics.datastore/desktop/2014/07 and later (the feature was only released in mid-July) # Parse the JSON for all events with feature ID "ti.apiusage". A sample item might appear like (note that API data itself is escaped) {code} { "event": "Titanium API Usage", "type": "ti.apiusage", "sid": "971063aa-1b36-470a-9933-a6d02679a975", "guid": "cf5c67ed-1c3b-494b-afe0-01b958ef0f40", "mid": "78f11f954dc9107fbe9f9b1b449f46b2", "mac_addr": "", "ip": "", "creator_user_id": "1559790", "app_name": "Titanium Command-Line Interface", "app_version": "3.3.0", "version": "3.3.0", "tz": "-120", "ver": "2", "un": "internet@createurdimage.fr", "data": "{\"platform\":\"iphone\",\"tisdkname\":\"3.3.0.v20140711123603\",\"tisdkver\":\"3.3.0\",\"deployType\":\"production\",\"target\":\"dist-appstore\",\"usage\":{\"Titanium.Locale.getCurrentLanguage\":10,\"Titanium.App.Properties.getString\":110,\"Titanium.App.Properties.setString\":9,\"Titanium.App.fireEvent\":50,\"Titanium.Platform.displayCaps.platformHeight\":3,\"Titanium.Platform.displayCaps.platformWidth\":159,\"Titanium.Platform.osname\":6,\"Titanium.Network.createHTTPClient\":63,\"Titanium.API.info\":61,\"Titanium.API.log\":31,\"Titanium.UI.createTableViewSection\":16,\"Titanium.UI.createAnimation\":64,\"Titanium.UI.createWindow\":19,\"Titanium.UI.createImageView\":60,\"Titanium.UI.createView\":169,\"Titanium.UI.FIll\":45,\"Titanium.UI.createButton\":37,\"Titanium.UI.createLabel\":96,\"Titanium.UI.Size\":50,\"Titanium.UI.SIZE\":99,\"Titanium.UI.createScrollView\":6,\"Titanium.UI.FILL\":24,\"Titanium.UI.createTableView\":13,\"Titanium.Filesystem.getFile\":11,\"Titanium.Filesystem.tempDirectory\":8,\"Titanium.UI.createOptionDialog\":7,\"Titanium.Media.showCamera\":7,\"Titanium.UI.createAlertDialog\":14,\"Titanium.Media.NO_CAMERA\":7,\"Titanium.Media.openPhotoGallery\":7,\"Titanium.Filesystem.applicationDataDirectory\":3,\"Titanium.App.addEventListener\":20,\"Titanium.UI.createTableViewRow\":8,\"Titanium.UI.Fill\":3,\"Titanium.UI.createTextField\":9,\"Titanium.UI.TEXT_AUTOCAPITALIZATION_NONE\":2,\"Titanium.UI.createTextArea\":2,\"Titanium.UI.createScrollableView\":1,\"Titanium.UI.createPicker\":4,\"Titanium.UI.PICKER_TYPE_DATE\":2,\"Titanium.Locale.getCurrentCountry\":1,\"Titanium.Contacts.getAllPeople\":1,\"Titanium.Contacts.contactsAuthorization\":2,\"Titanium.Contacts.AUTHORIZATION_AUTHORIZED\":1,\"Titanium.Contacts.AUTHORIZATION_UNKNOWN\":1,\"Titanium.Contacts.requestAuthorization\":1,\"Titanium.Platform.openURL\":1,\"Titanium.Platform.Android\":9,\"Titanium.UI.PICKER_TYPE_TIME\":1,\"Titanium.UI.PICKER_TYPE_DATE_AND_TIME\":1,\"Titanium.UI.PORTRAIT\":1,\"Titanium.UI.UPSIDE_PORTRAIT\":1,\"Titanium.UI.iPhone.TableViewSeparatorStyle.NONE\":2,\"Titanium.App.version\":2,\"Titanium.UI.createSearchBar\":1,\"Titanium.App.Properties\":1,\"Titanium.API.warn\":11,\"Titanium.Database.open\":6,\"Titanium.API.debug\":2,\"Titanium.Database.install\":1,\"Titanium.Database\":1,\"Titanium.version\":1,\"Titanium.UI.ActivityIndicatorStyle.DARK\":1,\"Titanium.UI.iPhone.ActivityIndicatorStyle.PLAIN\":1,\"Titanium.UI.createActivityIndicator\":1,\"Titanium.API.error\":1,\"Titanium.Platform.id\":2,\"Titanium.Platform.name\":1,\"Titanium.App.analytics\":1,\"Titanium.App.sessionId\":1,\"Titanium.App.deployType.toLowerCase\":1,\"Titanium.UI.createWebView\":1}}", "id": "7f26c3ed-cd66-5832-deeb-0b49-2227d851", "os": "Mac OS X", "platform": "osx", "osver": "10.9.4", "ostype": "64bit", "oscpu": "2", "memory": "4294967296", "node": "0.10.13", "npm": "1.3.2", "server_ts": "2014-08-01T08:01:10-07:00", "server_mid": "78f11f954dc9107fbe9f9b1b449f46b2" } {code} ## sid: Session ID ## guid: CLI App ID ## mid: Machine ID ## id: Event ID # Create a data storage mechanism that allows you to initially provide the following information: # Top 500 APIs in order of usage, descending. Each unique App ID only contributes once to the tally "jq" is a useful tool for parsing JSON. This strips out only the useful events: {code} ls -laS | grep ".*\.json.txt.gz" | head -n1 | awk '{print $NF}' | xargs gzip -dc | jq 'select(.type == "ti.apiusage") | [{id, data}] | group_by(.id)' | less {code} Once that's complete, provide a data storage that allows us to query the data easily to ask questions like: # Given a set of APIs, how many applications can be implemented 100%
Story points: 33