Kurt's Weblog

#!/usr/bin/env python

from numpy import array
import matplotlib.pyplot as plt
import sqlite3

xmin,xmax = 210000,220000

cx = sqlite3.connect('ais.db3')

pos_ok   = array( [tuple(row) for row in cx.execute('SELECT key,(est_sec) FROM position WHERE est_sec IS NOT NULL;')] ).T
pos_fail = array( [tuple(row) for row in cx.execute('SELECT key,(cg_sec) FROM position WHERE est_sec IS NULL;')] ).T

bs_ok    = array( [tuple(row) for row in cx.execute('SELECT key,.1 FROM bsreport WHERE est_sec IS NOT NULL;')] ).T

pos_sec  = array( [tuple(row) for row in cx.execute('SELECT key,timestamp FROM position WHERE timestamp < 60;')] ).T

fig = plt.figure()
ax1 = fig.add_subplot(2,1,1)
ax1.plot(pos_fail[0],pos_fail[1],'b+', pos_ok[0],pos_ok[1],'ro')

ax2 = ax1.twinx()
ax2.plot(bs_ok[0], bs_ok[1], 'gx')

ax2.set_ylim(0,10) #ymin, ymax  # For plotting on top of the same graph, but with a different axis
ax2.set_xlim(xmin,xmax)

ax3 = fig.add_subplot(2,1,2)
ax3.plot(pos_sec[0],pos_sec[1],'xb-')

ax3.set_xlim(xmin,xmax)

plt.show()

uname -a
Darwin CatBoxIII 9.7.0 Darwin Kernel Version 9.7.0: Tue Mar 31 22:52:17 PDT 2009; root:xnu-1228.12.14~1/RELEASE_I386 i386

uname -a
Darwin eel.ccom.nh 9.8.0 Darwin Kernel Version 9.8.0: Wed Jul 15 16:57:01 PDT 2009; root:xnu-1228.15.4~1/RELEASE_PPC Power Macintosh

softwareupdate --list

Software Update found the following new or updated software:
   * RemoteDesktopAdmin33-3.3
        Remote Desktop Admin Update (3.3), 57530K [recommended]
   * Safari403Leo-4.0.3
        Safari (4.0.3), 41443K [recommended] [restart]
   * MacOSXUpd10.5.8-10.5.8
        Mac OS X Update (10.5.8), 169217K [recommended] [restart]

xcodebuild -version
code 3.1.2
Component versions: DevToolsCore-1148.0; DevToolsSupport-1102.0
BuildVersion: 9M2621

sw_vers
ProductName:    Mac OS X
ProductVersion: 10.5.7
BuildVersion:   9J61

system_profiler | head
Hardware:

    Hardware Overview:

      Model Name: MacBook Pro
      Model Identifier: MacBookPro3,1
      Processor Name: Intel Core 2 Duo
      Processor Speed: 2.4 GHz
      Number Of Processors: 1
      Total Number Of Cores: 2

ioreg -l | grep IOPlatformSerialNumber
    |   "IOPlatformSerialNumber" = "W**********G"

hostname=`uname -n`
ip=`ifconfig en0 | grep inet | awk '{print $2}'`
serial=`system_profiler -detailLevel basic SPHardwareDataType | grep 'Serial Number:' | awk '{print $3}'`
model=`system_profiler -detailLevel basic SPHardwareDataType | grep 'Model Name:' | cut -c 7- `

fink -V | head -n2
Package manager version: 0.29.8
Distribution version: selfupdate-cvs Tue Aug 25 22:08:51 2009, 10.5, i386

dpkg -L tar | head
/.
/sw
/sw/bin
/sw/bin/gtar
/sw/lib
/sw/lib/rmt
/sw/sbin
/sw/share
/sw/share/doc
/sw/share/doc/tar

ps -a -w -w -x -O user | head
  PID USER       TT  STAT      TIME COMMAND
    1 root       ??  Ss     0:01.03 /sbin/launchd
   15 root       ??  Ss     0:00.98 /usr/libexec/kextd
   16 root       ??  Rs     0:11.31 /usr/sbin/DirectoryService
   17 root       ??  Ss     0:02.07 /usr/sbin/notifyd
   18 root       ??  Ss     0:01.03 /usr/sbin/syslogd
   19 root       ??  Ss     0:21.04 /usr/sbin/configd
   20 daemon     ??  Ss     0:03.82 /usr/sbin/distnoted
   21 _mdnsresponder   ??  Ss     0:01.55 /usr/sbin/mDNSResponder -launchd
   26 root       ??  Ss     0:00.45 /usr/sbin/securityd -i

uptime
23:30  up 3 days,  3:31, 1 user, load averages: 0.36 0.24 0.23

fink install nmap
nmap -A localhost
PORT      STATE SERVICE      VERSION
22/tcp    open  ssh          OpenSSH 5.1 (protocol 2.0)
80/tcp    open  http         Apache httpd 2.2.9 ((Unix) mod_python/3.3.1 Python/2.6.2 mod_wsgi/2.3)
88/tcp    open  kerberos-sec Mac OS X kerberos-sec
548/tcp   open  afp          Apple AFP (name: CatBoxIII; protocol 3.3; Mac OS X 10.5)
631/tcp   open  ipp          CUPS 1.3
5432/tcp  open  postgresql   PostgreSQL DB
6000/tcp  open  X11          (access denied)
8001/tcp  open  ssh          OpenSSH 5.1 (protocol 1.99)  Note: This is actually a tunnel to another machine
8080/tcp  open  http         Apache httpd 2.0.52 ((CentOS))  Note: This is actually a tunnel to another machine
50001/tcp open  unknown
1 service unrecognized despite returning data. If you know the service/version, please submit the following fingerprint 
Running: Apple Mac OS X 10.5.X
OS details: Apple Mac OS X 10.5 - 10.5.6 (Leopard) (Darwin 9.0.0 - 9.6.0)

sudo lsof | head -v 3
COMMAND     PID    USER   FD     TYPE     DEVICE  SIZE/OFF      NODE NAME
loginwind    40 schwehr  cwd      DIR       14,2      1394         2 /
loginwind    40 schwehr  txt      REG       14,2    946848  29312506 /System/Library/CoreServices/loginwindow.app/Contents/MacOS/loginwindow
loginwind    40 schwehr  txt      REG       14,2    600128  29320396 /System/Library/PrivateFrameworks/Admin.framework/Versions/A/Admin

top -o cpu -s 2
Processes:  110 total, 3 running, 1 stuck, 106 sleeping... 338 threads                            23:35:17
Load Avg:  0.11,  0.15,  0.18    CPU usage:  3.37% user,  6.07% sys, 90.56% idle
SharedLibs: num =    9, resident =   87M code, 2944K data, 5224K linkedit.
MemRegions: num = 14597, resident =  553M +   33M private,  364M shared.
PhysMem:  598M wired, 1413M active,  207M inactive, 2225M used, 1871M free.
VM: 11G + 376M   466472(1) pageins, 0(0) pageouts

  PID COMMAND      %CPU   TIME   #TH #PRTS #MREGS RPRVT  RSHRD  RSIZE  VSIZE
12986 firefox-bi   4.0%  7:13.11  14   207    981  112M+   27M-  158M   545M
    0 kernel_tas   3.8% 21:44.84  67     2    526 8048K      0   224M   351M
14113 top          3.7%  0:00.29   1    18     29  652K   200K  1248K    19M
   74 WindowServ   2.7% 12:12.73   5   321   1024  -15M+  107M+   93M   464M

df -h .
Filesystem     Size   Used  Avail Capacity  Mounted on
/dev/disk0s2  149Gi  139Gi  8.9Gi    95%    /

sysctl -a # Shows a whole lot of info
sysctl hw.ncpu # Name a specific variable and get the results (here number of cpus.

ALTER TABLE position ADD pkt_id INTEGER; 
ALTER TABLE position ADD dup_flag BOOLEAN;
ALTER TABLE bsreport ADD pkt_id INTEGER; 
ALTER TABLE bsreport ADD dup_flag BOOLEAN;

SELECT COUNT(*) duplicates FROM position WHERE dup_flag=1;
duplicates
3
SELECT COUNT(*) duplicates FROM position WHERE dup_flag=0;
duplicates
10
SELECT key,userid,longitude,latitude,pkt_id,dup_flag FROM position;

SELECT key,userid,Position_longitude as lon,Position_latitude as lat,pkt_id,dup_flag FROM bsreport;

"Our system, proven in orbit, can detect 1,000 ships in less than 90
seconds," said Mr Mabson.
...
COM DEV says that a satellite picking up AIS transmissions can see an
area of about 5,000km at any one time, covering that space for about
10 minutes.
...
"AIS has been an extremely successful maritime implementation, there
are now more than 70,000 AIS transponders on ships, the system is
rugged, there are many manufacturers of transponders and it's a
well-proven system," he said.

60 - time stamp is not available
61 - positioning system in manual input mode
62 - dead reckoning mode
63 - positioning system is inoperative)

./find_time.py
cg_t_min: 2008-04-07 03:57:20
cg_t_max: 2008-04-08 03:57:37
duration_mins: 1440.28
num_samples: 211830
ave_samples_per_min: 147.08
window_len: 88

$3 MILLION IN FED. STIMULUS MONEY TO FUND
BAY AREA UNDERSEA MAPPING PROJECT

California is receiving $3 million in federal stimulus money for an
undersea mapping project in the San Francisco Bay Area, federal
officials have announced.  ... The research surveys will map the sea
floor, measure water depth, identify storm debris and wreckage, and
identify coastal seabeds and fragile aquatic life, according to the
U.S. Commerce Department, which includes NOAA.  The Commerce
Department said the projects will create jobs, support marine commerce
and trade, and help coastal planning efforts.  The Bay Area surveys
will encompass about 112 square miles as part of a multi-year effort
to create the first comprehensive maps of the state's seafloor and
marine resources, according to the department.

CONTACT: Shannon Gilson, Commerce Department (202) 482-2237
         David Miller, NOAA (202) 482-0013
         John Ewald, NOAA (240) 595-0640

• ais_normalize now generates proper checksums. plus code cleanup
• ais_build_sqlite:
• Now validates checksums
• Msgs 1-4 now do commstate
• suggest at least one sql index
• Don't crash on short lines
• Added messge 19 for class B
• Better handling of uscg metadata. Still not handling them all
• Added ais msg 11 to decoding
• Msgs 1-4 are now hand coded to read comm state
• ais/__init__.py now has proper names for messages 11,13,16,17, and 22
• Added these uscg metadata fields to db: cg_s_rssi, cg_d_strength, cg_t_arrival, cg_s_slotnum
• aisutils/uscg.py: loosed regex to match bad T messages. Doced all field types
• Template is more better. Input from Trey Smith
• ais_liststations. cleanup and progress counts
• ais_position_in_polygon gives a better error message when pcl-core is missing. Still need to switch to shapely
• New scripts: ais_nmea_info.py, ais_nmea_remove_dups.py, ais_nmea_uptime.py, ais_sqlite_bsreport_analysis.py
• ais/commstate handles parsing of the SOTDMA and ITDMA commstates

!AIVDM,1,1,,B,403OwliuQ3SqiJs<SHH>jB100L0B,0*3C,s34022,d-075,T40.92853791,x123864,r003669945,1207540670
!AIVDM,1,1,,B,403OwliuQ3SqiJs<SHH>jB100L0B,0*3C,s28210,d-096,T11.02029935,x84922,r003669947,1207540645
!AIVDM,1,1,,B,403OwliuQ3SqiJs<SHH>jB100L0B,0*3C,s29329,d-096,T50.26997285,x64858,r003669946,1207540666

ais_build_sqlite.py -d trouble.db3 trouble.ais -C
{1: 14, 2: 0, 3: 0, 4: 5, 5: 0, 18: 0, 19: 0}

sqlite3 -html trouble.db3 \
  'SELECT key, UserID as MMSI, \
          Time_day as day, Time_hour as hour, Time_min as min, Time_sec AS sec, \
          sync_state, slot_timeout, received_stations, cg_timestamp, \
          cg_t_arrival, cg_r \
   FROM bsreport WHERE userid=3669971;'

SELECT commstate_utc_hour, commstate_utc_min as com_min, timestamp as sec, cg_timestamp, 
	cg_t_arrival, cg_r 
    FROM position 
    WHERE commstate_utc_hour IS NOT NULL;

#!/bin/bash
for file in *20[0-9][0-9]-??-??.bz2; do 
    basename=`basename $file`
    basename=${basename%%.bz2}
    basename=${basename##prefix}
    echo $basename
    tmp=$basename.norm

    bzcat $file | ais_normalize.py > $tmp

    # Lookback:  125 stations * 1000 messages = 125000
    egrep 'AIVDM,1,1,[0-9]?,[AB],5' $tmp | ais_nmea_remove_dups.py -v -l 100000 > $basename.5.unique
done

!AIVDM,1,1,9,A,576uEH02>rgMI8AON20PTLR1<5AE8r222222221@B`R:F5Jo0AS3lhCQiC1DvmDhH888880,2*35,s25682,d-105,T30.27102750,x179634,r11CSDO1,1242917069

cat *.5.unique |  ais_nmea_remove_dups.py -v -l 100000 > 5.unique.nmea
ais_build_sqlite.py -C -d 5.unique.db3 5.nmea
{1: 0, 2: 0, 3: 0, 4: 0, 5: 92895, 18: 0, 19: 0}

sqlite3 5.unique.db3 'SELECT shipandcargo,count(*) FROM shipdata WHERE shipandcargo in (40,60,61,62,63,64,65,66,67,68,69) GROUP BY shipandcargo;'
shipandcargo    count(*)
40      535
60      5876
61      104
62      67
63      47
64      278
65      10
66      45
67      40
68      243
69      3374

sqlite3  5.unique.db3 < /Users/schwehr/projects/src/noaadata/data/mmsi_prefix.sql

SELECT userid AS mmsi, name, callsign, imonumber, shipandcargo,
       dima+dimb AS width, dimc+dimd AS length, synonym AS flag 
       FROM shipdata
       LEFT JOIN mmsi_prefix ON mmsi_prefix.code==shipdata.userid/1000000 
       WHERE shipandcargo IN (40,60,61,62,63,64,65,66,67,68,69) 
       GROUP BY mmsi,name;

1193046,"SAMUEL S COURSEN@@@@",WDB3537,303174162,60,55,19,Null
1193046,"SAMUEL S CURSEN@@@@@",WDB3537,303174162,60,55,19,Null
3160015,"LADY WHITE HEAD     ","VC8150 ",7947960,60,34,10,Null
4490052," @LWHA              ","WY397P ",8835358,60,116,22,Null
31602656,"TANDEM              ",CFB5593,813929,69,20,7,Null
35516535,"P/B WANDERER        ",@@@@@@@,0,64,15,6,Null
35517401,"P/B PHANTOM         ",@@@@@@@,0,69,20,6,Null
36677150,"NAPA                ",WDE8110,0,60,38,10,Null
100000000,"BRITANNIA           ","VY5348 ",0,60,0,0,Null
100000001,@@@@@@@@@@@@@@@@@@@@,@@@@@@@,100000001,60,53,7,Null
107502700,"ROBERT H. DEDMAN    ",WCY9892,920700300,69,83,23,Null
111111111,"BLUE ICE            ",WCW8492,1111111,60,0,0,Null
111850000,"FAST BULLET         ",WDA8488,366854890,60,50,10,Null
123456789,"GOOSE BUMPS         ","KUME   ",123456787,60,30,6,Null
123456789,"MADDEN TIDE@@@@@@@@@",WCX9599,0,60,0,0,Null
203888479,@@@@@@@@@@@@@@@@@@@@,@@@@@@@,0,61,0,0,Austria
235762000,"QUEEN MARY 2        ","GBQM   ",9241061,60,345,41,"United Kingdom"
244078000,STATENDAM@@@@@@@@@@@,PHSG@@@,8919245,69,220,30,Netherlands
...

SELECT code,synonym FROM mmsi_prefix 
   WHERE code IN (303,316,338,358,366,367,368,369,379);
code    synonym
303     Alaska
316     Canada
338     United States
358     Puerto Rico
366     United States
367     United States
368     United States
369     United States
379     United States Virgin Islands

SELECT userid AS mmsi, name, callsign, imonumber, shipandcargo,
       dima+dimb AS width, dimc+dimd AS length, synonym AS flag 
       FROM shipdata 
       LEFT JOIN mmsi_prefix ON mmsi_prefix.code==shipdata.userid/1000000 
       WHERE shipandcargo IN (40,60,61,62,63,64,65,66,67,68,69) 
           AND userid/1000000 IN (303,316,338,358,366,367,368,369,379) 
       GROUP BY mmsi,name;

 mmsi,name,callsign,IMOnumber,shipandcargo,width,length,flag
303267000,"TUSTUMENA","WNGW",6421086,60,90,18,Alaska
303472000,"ISLAND CAPER",WAM9248,0,60,29,6,Alaska
303488000,"BERNIE MCALL",WDC6875,936351000,60,0,0,Alaska
303509000,"SPIRIT OF YORKTOWN",WDD5898,8949472,60,68,12,Alaska
303556000,"MARITIME INSTRUCTOR",WCW8625,1033659,60,26,5,Alaska
303677000,"MISS PAMELA",WDC3158,9111254,60,40,9,Alaska
303913000,"GORDON GUNTER","WTEO",8835255,60,205,32,Alaska
316001111,"NAUTILUS EXPLORER",CFG7418,9224647,60,35,8,Canada
316001232,"BOWEN QUEEN","VGXR",6600967,69,88,18,Canada
316001234,"HOWE SOUND QUEEN","CY3842",801691,60,80,20,Canada
316001235,"KLITSA","CZ3072",0,60,52,13,Canada
...
338086000,"MISS CARISSA",WDB7801,8977699,69,39,8,"United States"
338088000,"MISS BETH",WDB7803,8977687,60,36,9,"United States"
338092962,"SEABRIGHT",,0,67,20,5,"United States"
338116000,B AP3D P AP,SD U<P4,9265809,40,36,14,"United States"
338116000,FAIRWEATHER,WDB5604,9265809,40,72,22,"United States"
338117000,"JENNY MCCALL",WDC3101,931679900,60,54,11,"United States"
338163000,COPACABANA,WDE6748,1214699,60,53,9,"United States"
338194000,"PHILIP ALAN MCCALL",WDB7424,108745800,60,51,10,"United States"
338195000,"MATANUSKA","WN4201",5228827,69,124,22,"United States"
...

316011408,"COASTAL INSPIRATION",CFN4970,9332767,69,160,28,Canada
316011408,"COAUTAL INSPIRATION",CFN4970,9332767,69,160,28,Canada

338116000,B AP3D P AP,SD U<P4,9265809,40,36,14,"United States"

CREATE TABLE position ( 
	key INTEGER PRIMARY KEY,
	MessageID INTEGER,		-- 1 or 2 for SOTDMA or 3 for ITDMA
	RepeatIndicator INTEGER,	-- 0 means good.  1-3 means the message has been repeated (DANGER!)
	UserID INTEGER,			-- MMSI
	NavigationStatus INTEGER,
	ROT INTEGER,
	SOG REAL,
	PositionAccuracy INTEGER,
	longitude REAL,
	latitude REAL,
	COG REAL,
	TrueHeading INTEGER,
       	sec_timestamp INTEGER,
	--RegionalReserved INTEGER,	-- Replaced by the next field
        special_manoeuvre INTEGER,	-- 0 N/A, 1: not engaged, 2: engaged
	Spare INTEGER,
	RAIM BOOL,
	sync_state INTEGER,		-- 0 UTC direct, 1 UTC indirect, 2 sync to bs, 3 sync to mobile to bs
	slot_timeout INTEGER,		-- frames until new slot.  Controls comstate type
	received_stations INTEGER,	-- SOTDMA: # of stations this receiver hears
	slot_number INTEGER,		-- SOTDMA: slot number used for transmission of this message
	commstate_utc_hour INTEGER,	-- SOTDMA: Hour of the day
	commstate_utc_min INTEGER,	-- SOTDMA: UTC minutes
	commstate_utc_spare INTEGER,    -- SOTDMA: Should be zero
	slot_offset INTEGER,		-- SOTDMA: Offset to next slot used for next transmission. 0 if done talking
	slot_increment INTEGER,		-- ITDMA:  Offset to next slot for ITDMA.  0 if done talking
	keep_flag INTEGER,		-- ITDMA:  1 means keep the slot allocated for an additional frame.
	slots_to_allocate INTEGER,	-- ITDMA:  Slots to allocate.

        -- USCG metadata fields
        cg_s_rssi INTEGER,		-- 's' Receive strength indicator
        cg_d_strength INTEGER,		-- 'd' dBm receive strenth
        cg_x_count INTEGER,		-- 'x' Count of messages received by that station
	cg_t_arrival INTEGER,	        -- 't' HHMMSS.SS receive time
	cg_s_slotnum INTEGER,		-- 'S' Slot number of receive.  9999 means?
	cg_r VARCHAR(15),		-- Receiver or basestation name
	cg_sec INTEGER,			-- Unix UTC receive timestamp to second resolution
	cg_timestamp TIMESTAMP 		-- UTC time of receive to second resolution
);

char *results = (retval ? "true string" : "false string");

#!/usr/bin/env python

import sys

def good():
    return True

def bad():
    return False

print >> sys.stderr, 'trying good...\t',
r = good()
print >> sys.stderr, 'ok' if r else 'FAILED'

print >> sys.stderr, 'trying bad...\t',
r = bad()
print >> sys.stderr, 'ok' if r else 'FAILED'

% results.py
trying good...  ok
trying bad...   FAILED

2. USCG National AIS Project Increment 3 - Briefer: Mr E G Lockhart
3. Volpe MSSIS/IALA.net - Briefer: Dave Phinney, Volpe Center
4. Naval Research Lab (Gladis) - Briefer: Jim Tugman, NRL
5. Analysis of Spacequest acquired AIS signals - Briefer: Dino Loenzini, Spacequest
6. Status of AIS Frequencies Nationally/Internationally - Briefer: Joe Hersey, USCG
7. Technical Limitations and Possible Solutions - Briefer: Ross Norsworthy
8. Report on AIS via COMDEV Briefer: George - Best, COMDEV
10. AIS Analyses results and standards - Briefer: Dave Pietraszewski, USCG R&D Ctr
10-2. Status of Ship Wakes - Briefer: Jake Tunaley USCG R&D Ctr
12. Radar Sat Processing Tech - Brief Briefer: Dr Jake Tunaley
13-2. TerraSar Basic Product Spec Document
14. OGMSA overview - Briefer: Guy Thomas, OGMSA

SUMMARY: The Coast Guard is establishing two temporary safety zones
extending 500 meters in all directions from each of the two submerged
turret- loading buoys and accompanying systems that are part of GDF
Suez Energy's Neptune Deepwater Port located in the Atlantic Ocean off
of Boston, Massachusetts. The purpose of these temporary safety zones
is to protect vessels and mariners from the potential safety hazards
associated with construction of the deepwater port facilities and the
large sub-surface turret buoys, and to protect the deepwater port
infrastructure. All vessels, with the exception of deepwater port
support vessels, are prohibited from entering into, remaining or
moving within either of the safety zones.

DATES: This rule is effective from July 31, 2009 through February 16,
2010.
...

sqlite3 foo.db
.header ON
.separator "\t"
.nullvalue "Null"

( \
  echo "<table>"; \
  sqlite -html 20090506-2.db3 'SELECT name,shipandcargo FROM shipdata LIMIT 10;' \
     | grep -v 'Loading resources' \
     | tr '@' ' '; \
  echo "</table>" \
)  > foo.html

-- Loading resources from /Users/schwehr/.sqliterc

ipython
from BitVector import BitVector
bv = BitVector(intVal=121)
print str(bv)
'1111001'

print(int(BitVector(bitstring='11101')))
29

print(int(BitVector(bitstring='11110')))
30

NoBits MaxVal   Bits    Unavail   Bits    Loss-of-Range
 5       31       11111   30        11110 3%
 6       63      111111   60       111100 5%
 7      127     1111111  122      1111010 4%
 8      255    11111111  246     11110110 4%
 9      511   111111111  494    111101110 3%
10     1023  1111111111  990   1111011110 3%

time bzip2 -9 b
real    17m19.233s
user    16m41.131s
sys     0m8.296s

time pbzip2 -9 -p7 a
real    2m45.526s
user    18m33.926s
sys     0m13.202s

842988852 Aug 12 23:19 a.bz2
841630033 Aug 12 23:19 b.bz2

...
On July 28, the Arctic Sea made contact with British maritime
authorities as it passed through the busy English Channel. The ship
made a routine, mandatory report - saying who they were, where they
were from, where they were going and what their cargo was. It appeared
routine, said Mark Clark of Britain's Maritime and Coastguard Agency.
...
Where the ship was next spotted is uncertain. Russian media reports
say the last contact was on July 30 when the ship was in the Bay of
Biscay, and that it was later spotted by a Portuguese patrol plane,
but there was no contact.

But Portuguese Navy spokesman Commander Joao Barbosa said "we can
guarantee that the ship is not in Portuguese waters nor did it ever
pass through Portuguese waters."
...

fink install matplotlib-py26 ipython
ipython -pylab
import sqlite3
cx = sqlite3.connect('20080407.decimated.db3')
a = array( [tuple(row) for row in cx.execute('SELECT cg_offset, delta_t_sec FROM bs_time WHERE recvr=2;')] )
x = a[:,0]
y = a[:,1]
plot(x,y)

a.shape
(50756, 2)
a.T.shape
(2, 50756)
a.T
array([[  2.40000000e+01,   2.50000000e+01,   3.40000000e+01, ...,
          8.63840000e+04,   8.63940000e+04,   8.64040000e+04],
       [  0.00000000e+00,  -1.00000000e+00,   0.00000000e+00, ...,
          2.00000000e+01,   2.00000000e+01,   2.00000000e+01]])

plot(*array( [tuple(row) for row in cx.execute('SELECT cg_offset, 
      delta_t_sec FROM bs_time WHERE recvr=2;')] ).T)

...
Crews on board NOAA ships Fairweather and Rainier will conduct
hydrographic surveys of the seafloor, measuring depths and identifying
obstructions. When the ships complete data collection in early
September, they will have surveyed more than 350 square nautical miles
- an area nearly twice the size of Chicago.

"This is one of our largest survey efforts of the year," said
Capt. Steven Barnum, director of NOAA's Office of Coast Survey and
U.S. national hydrographer. "The data we collect will contribute to
navigation safety in the state and also be used to keep the coast
healthy and productive."
...
"Our data are used by countless federal, state, and local
stakeholders,¡ said Captain Roger Parsons, NOAA Corps (ret.), IOCM
director. ´Coastline and seafloor data, once collected solely for
updating NOAA's national suite of nautical charts, is now used for
marine spatial planning efforts, ocean circulation monitoring, and
assessing the impacts of climate change, among other uses."
...

sudo apt-get install libjpeg-dev libpng-dev libblas-dev autoconf libtool \
  build-essential iblapack-dev libboost-dev imagemagick libexpat1-dev python-numpy \
  apache2 libapache2-mod-wsgi libreadline-dev flex # For GeoCam
sudo apt-get install devscripts build-essential wget fakeroot cdbs patchutils debhelper # debian install tools

mkdir -p deb_pkgs/jquery
cd !$
dget -xu https://launchpad.net/ubuntu/jaunty/+source/jquery/1.2.6-2ubuntu1/+files/jquery_1.2.6-2ubuntu1.dsc
# apt-get source jquery ### Instead of dget, you can do this line
cd ..
mkdir hello-debhelper
cd !$
apt-get source hello-debhelper
cd ..

mkdir openlayers-2.8
cd !$
wget http://openlayers.org/download/OpenLayers-2.8.tar.gz
cp OpenLayers-2.8.tar.gz openlayers_2.8.orig.tar.gz
mkdir -p debian/openlayers-2.8
cd debian/openlayers-2.8
dh_make -e kurtschwehr@yahoo.com # Puts a ton of stuff in the debian directory
cd debian
rm *.ex *.EX # Remove un-need template files

#!/usr/bin/make -f
# -*- makefile -*-
# Uncomment this to turn on verbose mode.
#export DH_VERBOSE=1

build: 
	(cd build; ./build.py)
	rm -f tools/*.pyc # Fails to build without this cleanup

clean: 
	dh_testdir
	dh_testroot
	dh_clean 

install:

binary-indep: install
	dh_testdir
	dh_testroot
	dh_installchangelogs
	dh_installdocs
	dh_install
	dh_link
	dh_compress
	dh_fixperms
	dh_installdeb
	dh_gencontrol
	dh_md5sums
	dh_builddeb

binary-arch:

binary: binary-indep

.PHONY: build clean binary-indep binary-arch binary install

Source: openlayers
Section: web
Priority: extra
Maintainer: Kurt Schwehr 
Depends: ${misc:Depends}
Suggests: javascript-common
Build-Depends: debhelper (>= 6)
Standards-Version: 3.8.0
Homepage: http://openlayers.org/

Package: libjs-openlayers
Architecture: all
Description: JavaScript web mapping
  OpenLayers makes it easy to put a dynamic map in any web page. It
  can display map tiles and markers loaded from any source. MetaCarta
  developed the initial version of OpenLayers and gave it to the
  public to further the use of geographic information of all
  kinds. OpenLayers is completely free, Open Source JavaScript,
  released under a BSD-style License.

gpg --gen-key # Use all the default values
gpg --output kurtschwehr.gpg --export kurtschwehr@yahoo.com # Put in your name and email here
scp kurtschwehr.gpg schwehr.org:www/ # Put your public key somewhere for people be able to get it

cd ~/deb_pkgs/openlayers-2.8/openlayers-2.8
debuild
# enter your password 2 times to sign the package
ls -l libjs-openlayers_2.8-1_all.deb

debuild -S
ls -l ../openlayers_2.8-1.diff.gz

sudo pbuilder --create # Takes a while
sudo pbuilder build ../*.dsc

pbuilder-satisfydepends-dummy: Depends: debhelper (>= 7) but it is not installable

sudo dpkg -i libjs-openlayers_2.8-1_all.deb

A Coast Guard Cutter Healy boatcrew along with a scientist from
Scripps Institution of Oceanography recovered a sonobuoy, Aug. 8,
2009, while on a scientific research mission in the Arctic Ocean.
...

...
But this happy meeting of robotic beings hides a serious problem:
while the robots might be collaborating, those making them are
not. Each robot is individually manufactured to meet a specific need
and more than likely built in isolation.

This sorry state of affairs is set to change. Roboticists have begun
to think about what robots have in common and what aspects of their
construction can be standardised, hopefully resulting in a basic
operating system everyone can use. This would let roboticists focus
their attention on taking the technology forward.
...

fink rebuild dpkg # Force fink to download the source
mkdir ~/foo && cd ~/foo
cp /sw/fink/10.4/unstable/main/finkinfo/base/dpkg.patch . # Pull in from fink

tar xf /sw/src/dpkg_1.10.21.tar.gz
sed 's|@PREFIX@|/Users/schwehr/foo/world|g' < dpkg.patch | patch -p0
mkdir build && cd build
rootdir=/Users/schwehr/foo/world
mkdir $rootdir
../dpkg-1.10.21/configure --prefix=$rootdir  --without-start-stop-daemon --without-sgml-doc \
  --with-admindir=$rootdir/var/lib/dpkg --mandir=$rootdir/share/man \
  --infodir=$rootdir/share/info --srcdir=$rootdir/../dpkg-1.10.21
make  all-include all-optlib all-lib all-dpkg-deb all-split all-scripts all-utils \
  all-main all-dselect all-po all-methods all-man all-doc -j 3

make # Seems to then work?  Will this blow up later?

make install DEST=somewhere

...
Larry Mayer, director of the Center for Coastal and Ocean Mapping and
co-director of the Joint Hydrographic Center, is the chief scientist
for the U.S. mission. NOAA's Andy Armstrong, a physical scientist and
co-director of the Joint Hydrographic Center, is the co-chief
scientist. NOAA and the University of New Hampshire jointly operate
the Joint Hydrographic Center.

The 41-day joint mission runs from August 7 to September 16 and will
see the U.S. Coast Guard Cutter Healy and the Canadian Coast Guard
Ship Louis S. St-Laurent operating together to obtain a variety of
data.

"NOAA and the Joint Hydrographic Center will take the lead in
collecting bathymetric data from the Healy to map the seafloor, while
the Canadian icebreaker collects seismic data to determine sediment
thickness," said Craig McLean, deputy assistant administrator for
NOAA's Office of Oceanic and Atmospheric Research. "This
collaboration saves millions of dollars by ensuring data are collected
only once in the same area, and by sharing data useful to both
nations."
...

At present the following automatic ship reporting systems are either
adopted or proposed for adoption by IMO and FAO, including:

-	Global Maritime Distress and Safety System (GMDSS)
-	Automatic Identification System (AIS)
-	Long-range AIS (LR-AIS)
-	Long Range Ship's Identification and Tracking (LRIT)
-	Vessel Monitoring System for fisheries (VMS)
-	Ship security alert system (SSAS)
-	'Daily Report'

In addition, a number of commercial fleet management services based on
GPS positioning are in operation, for the most part based on
Inmarsat-C position reports.

While AIS Class A and Class B will meet the need of VTM for detailed
automated ship reporting within the coastal areas and in-land waters
covered by VHF shore stations, no such system has been adopted or
proposed on a global scale.  The recently adopted LRIT system will, in
its present form, only provide identity, position and time information
and basically cover maritime security applications.

The future take-up of AIS Class B by Non-SOLAS and leisure vessels
remains to be seen.  However, with a price tag of around 500 Euros and
offered as an optional item to VHF radios, it is likely that Class B
transponders will be well accepted by at least the larger leisure
vessels.

However, there are some questions raised with respect to the
capability of AIS shore stations to handle the non-priority Class B
traffic in some areas with an adequate reporting rate to be of any use
to both the VTM authorities and the mariners. Such deficiencies may be
augmented using other available or proposed terrestrial communication
systems, i.e. WIMAX, DVHF or GSM/UMTS.  Solutions based on WIMAX using
a peer-to-peer methology may also offer ranges beyond VHF and GSM and
possibly serve as a 'gap-filler' between AIS and LRIT.
...

% ls ~/Library/Mail/Mailboxes/Notes.mbox/Messages
29343.emlx  29345.emlx  29347.emlx  29349.emlx  29351.emlx  29353.emlx
29344.emlx  29346.emlx  29348.emlx  29350.emlx  29352.emlx  29416.emlx

616
-Uniform-Type-Identifier: com.apple.mail-note
Message-Id: <693D209C-19BF-4687-BD99-DAC020E4ECDD@gateway.2wire.net>
From: =?MACINTOSH?Q?Kurt=D5s_iPhone?=
Content-Type: text/html;
	charset=US-ASCII
-Universally-Unique-Identifier: 86207fd8-541f-434f-a374-84a9e6f735d9
Content-Transfer-Encoding: 7bit
-Mail-Generated-Subject: YES
-Mail-Created-Date: 2009-07-19 15:47:26 -0700
Mime-Version: 1.0 (Apple Message framework v935.3)
Subject: =?MACINTOSH?Q?=CAerrands?=
Date: 2009-07-24 15:51:48 -0700

mongolian beef<div>muffins&nbsp;</div><div>waffles</div><div>milk</div><div><br><div><br></div></div>
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
	<key>date-sent</key>
	<real>1248475908</real>
	<key>flags</key>
	<integer>33751169</integer>
	<key>sender</key>
	<string>Kurt's iPhone</string>
	<key>subject</key>
	<string>errands</string>
	<key>to</key>
	<string></string>
</dict>
</plist>

#!/usr/bin/env python

import email
import sys

em_file = file(sys.argv[1])
em_file.readline()
em = email.message_from_file(em_file)

content = []
for line in em.get_payload().splitlines():
    if '<?xml version="1.0" encoding="UTF-8"?>' == line.strip():
        break
    content.append(line)

results = ''.join(content)
print results

% ./notes.py 29344.emlx | html2text
mongolian beef
muffins 
waffles
milk

Welcome to the Website for the U.S. Extended Continental Shelf
Project! The 2009 U.S.-Canada Arctic Continental Shelf Survey runs
from August 7 through September 16.

Despite it's widespread adoption, GeoRSS has some complexities that
arose out of it's development. There are 9 potential 'flavors' of
GeoRSS, although this is largely due to the 3 different formats of
feeds: RDF, RSS, and Atom. There are still 3 formats of GeoRSS itself
that can be utilized in any of the 3 feed formats: W3C, Simple, and
GML. This causes confusion for developers, especially since W3C format
is deprecated but still widely used. Perhaps this is one reason that
despite GeoRSS being a simple extension to existing feed formats,
there still is not GeoRSS support in any of the major news feed
readers, except perhaps limited support in FriendFeed.

In addition, GeoRSS hasn't really advanced in quite awhile despite
multiple requests and discussions of extensions for multiple
locations, time spans, external geometries, and feature
identification.

Three lobster boats were sabotaged and two of them sunk in a
midcoast harbor amid growing tensions among lobstermen two weeks after
another lobster turf war escalated into a shooting on Matinicus
Island, law enforcement officials said Wednesday.
...

Designed for use by clients and contractors alike, the International
Marine Contractors Association (IMCA) 'Code of Practice for the Safe
and Efficient Operation of Remotely Operated Vehicles (ROVs)' has been
reviewed and updated to reflect technological and operational
developments, and includes additional guidance provided in particular
on safe crewing levels for ROVs equipped with the growing range of
tools packages in use within this fast-developing industry.
...

...
The NOAA Marine Operations Center-Pacific is comprised of
approximately 175 employees, including more than 110 officers and crew
assigned to the NOAA ships McArthur II, Miller Freeman, Rainier and
Bell M. Shimada, a new fisheries survey vessels expected to join the
NOAA fleet in 2010.
...

uscg_ais_nmea_regex_str = r"""[!$](?P<talker>AI)(?P<stringType>VD[MO])
,(?P<total>\d?)
,(?P<senNum>\d?)
,(?P<seqId>[0-9]?)
,(?P<chan>[AB])
,(?P<body>[;:=@a-zA-Z0-9<>\?\'\`]*)
,(?P<fillBits>\d)\*(?P<checksum>[0-9A-F][0-9A-F])
(  
  (,S(?P<slot>\d*))
  | (,s(?P<s_rssi>\d*))
  | (,d(?P<signal_strength>[-0-9]*))
  | (,t(?P<t_unknown>[^,]*))
  | (,T(?P<time_of_arrival>[0-9.]*))
  | (,x(?P<x_station_counter>[0-9]*))
  | (,(?P<station>(?P<station_type>[rbB])[a-zA-Z0-9]*))
)*
,(?P<timeStamp>\d+([.]\d+)?)?
"""

import re
try:
    set
except NameError:
    from sets import Set as set

from pygments.lexer import RegexLexer
from pygments.token import Text, Name, String, Literal

__all__ = ['NmeaLexer']

class NmeaLexer(RegexLexer):
    '''Data streams from GPS, AIS, and other sensors.
    Does not support the NMEA 4.0 TAG Block extensions
    '''

    name = 'Nmea'
    aliases = ['nmea', 'NMEA', 'GPS', 'AIS', 'ais']
    filenames = ['*.ais',] # And?

    flags = re.VERBOSE

    tokens = {
        'root': [
            
            (r'[!$][^,]*', String), #,'fields'), # Talker and message type
            (r',[^,*\n]*', Text), # Field
            #(r',
            (r'[*][0-9A-F][0-9A-F]', Name, 'uscg-metadata'), # Checksum and move into USCG metadata
            ],
        'uscg-metadata': [
            (r',[^drstTx0-9][^,]*',Name.Entity), # Unknown things
            (r',d[^,]*',Name.Builtin), # signal strength
            (r',r[^,]*',Name.Attribute), # Station id/name
            (r',s[^,]*',Name.Class), # RSSI
            (r',t[^,]*',Name.Constant), # unknown 
            (r',T[^,]*',Name.Decorator), # time of arrival
            (r',x[^,]*',Name.Function), # station counter
            (r',[0-9]*',Name.Constant,'#pop'),
            (r'$',Name.Tag,'#pop'), # Make sure we pop.  Doesn't see to help
            ]
        }

    'NmeaLexer': ('pygments.lexers.nmea', 'Nmea', ('nmea',), ('*.ais',), ()),

% grep ',120756572' 20080407.log.norm | grep 8LvSh5000Jr
!AIVDM,1,1,,B,38LvSh5000JrdUrH@D<sk8jt0PUA,0*6B,s24733,d-110,T42.44911276,x16510,r003669947,1207565720
!AIVDM,1,1,,B,38LvSh5000JrdUtH@D<ck8jv0PSQ,0*69,s24730,d-110,T42.47580536,x16511,r003669947,1207565720
!AIVDM,1,1,,A,38LvSh5000JrdUrH@D<sk8k00PVi,0*06,s23257,d-117,T44.31575392,x16513,r003669947,1207565722
!AIVDM,1,1,,A,38LvSh5000JrdUrH@D<sk8k20PU1,0*5F,s23271,d-117,T44.47572724,x16514,r003669947,1207565722
!AIVDM,1,1,,B,38LvSh5000JrdUrH@D<sk8k40PVi,0*01,s24719,d-110,T46.10240842,x16515,r003669947,1207565724
!AIVDM,1,1,,B,38LvSh5000JrdUrH@D<sk8k40PUA,0*2A,s24698,d-111,T46.42240714,x16516,r003669947,1207565724
!AIVDM,1,1,,B,38LvSh5000JrdUpH@D=;k8k60PTQ,0*72,s24611,d-111,T46.52908640,x16517,r003669947,1207565724
!AIVDM,1,1,,B,38LvSh5000JrdUpH@D=;k8k60PTQ,0*72,s24609,d-111,T46.60907306,x16518,r003669947,1207565725
!AIVDM,1,1,,B,18LvSh5000JrdUnH@D=;k8k<0d0B,0*27,s24396,d-112,T50.23571518,x16522,r003669947,1207565728
!AIVDM,1,1,,B,18LvSh5000JrdUnH@D=;k8k<0d0B,0*27,s24326,d-112,T50.42236808,x16523,r003669947,1207565728

% ais_msg_1.py -d '!AIVDM,1,1,,B,18LvSh5000JrdUnH@D=;k8k<0d0B,0*27,s24326,d-112,T50.42236808,x16523,r003669947,1207565728'
position:
        MessageID:          1
        RepeatIndicator:    0
        UserID:             567256000
        NavigationStatus:   5
        ROT:                0
        SOG:                0
        PositionAccuracy:   0
        longitude:          -71.06358167
        latitude:           42.388566667
        COG:                302
        TrueHeading:        281
        TimeStamp:          38
        RegionalReserved:   0
        Spare:              0
        RAIM:               False
        state_syncstate:    1
        state_slottimeout:  3
        state_slotoffset:   18

% sqlite ais.db3
SELECT imonumber,callsign,name,shipandcargo,dima+dimb,dimc+dimd,fixtype,destination FROM shipdata WHERE userid=567256000 GROUP BY imonumber,callsign,name,shipandcargo LIMIT 10;
IMOnumber callsign  name          shipandcargo dima+dimb  dimc+dimd  fixtype destination
8312174   HSBE      THOR ALLIANCE 70           189 m      31 m       1       BOSTON,USA

o = file('unique.ais','w')

dropped = 0
for line_num, line in enumerate(file('20080407.log.norm')):
    if line_num % 10000 == 0:
        print 'line_num:',line_num, '  dropped:',dropped
    if '!AIVD' != line[:5]:
        o.write(line)
    fields = line.split(',')
    payload = fields[5]
    if payload in payloads:
        dropped += 1
        continue
    if len(payloads) > lookback_dist:
        payloads.pop(0)
    payloads.append(payload)
    o.write(line)

The Ocean Survey Vessel Bold (The Bold) is EPA's only ocean and
coastal monitoring vessel. The Bold is equipped with state-of-the-art
sampling, mapping, and analysis equipment including sidescan sonar,
underwater video, water sampling instruments, and sediment sampling
devices, which scientists can use in various monitoring
activities. The vessel is a converted U.S. Navy T-AGOS class vessel
and is 224 feet long and 43 feet wide. EPA acquired the Bold on March
31, 2004.

...
U.S. Sen. Jeanne Shaheen, D-N.H., told staff following a tour of the
center that the Senate has approved $750,000 for their work as part of
the Energy and Water Appropriations bill. Although the bill still has
to go to a conference committee, she said she's confident the funding
will pass through unscathed.
...