def F1(p):
    '''Number of points within unit circle'''
    d = np.sqrt(np.sum(p ** 2, axis=1))
    return np.sum(d <= 1)

def F2(p, d):
    '''Number of points with at least one neighbor within distance d.'''
    count = 0
    for i, p1 in enumerate(p):
        for j, p2 in enumerate(p):
            if i != j:
                d12 = np.sqrt(np.sum((p1 - p2) ** 2))
                if d12 <= d:
                    count += 1
                    break
    return count

def F3(rain):
    '''Largest number of consecutive days of rain.'''
    longest = 0
    current = 0
    for r in rain:
        if r > 0:
            current += 1
            if current > longest:
                longest = current
        else:
            current = 0
    return longest

def F4(table):
    '''Average rainfall for each month.'''
    result = np.zeros(12)  # you are to return an array of 12 means
    for i in range(12):
        result[i] = np.mean(table[table[:, 1] == (i + 1), 3])
    return result

def F5(rain):
    '''Number of weeks with no rain.'''
    rain.shape = (-1, 7)  # reshape into weeks
    weekly = np.sum(rain, axis=1)
    dry = weekly == 0
    dryweeks = np.sum(dry)
    return dryweeks

First to remove the lines with duplicate codes.

# create an empty dictionary to hold the codes we've seen
codes = {}

# create an output file
ofile = file('outlines.txt', 'w')

# read and process each line from the input
for line in file('lines.txt'):
    # get the code
    code = line[:10]  # get the code
    if code not in codes:  # have we seen it before?
        # no, write this line and remember the code
        ofile.write(line)
        codes[code] = True

# close the output file
ofile.close()

Then to translate text messages.

# initialize our dictionary of acronyms
tla = {
    'abt': 'about',
    'adn': 'any day now',
    'irl': 'in real life',
    'lol': 'laugh out loud',
    'b4': 'before'
}

def translate(text):
    result = []
    for word in text.split():
        if word in tla:
            result.append(tla[word])
        else:
            result.append(word)
    return ' '.join(result)

print translate('I am abt to lol irl')

def F1(table):
    '''Number of subjects more than 10 pounds heavier than the mean'''
    w = table[:, 3]
    m = np.mean(w)
    n = np.sum(w > (m + 10))
    return n

def F2(table):
    '''Mean BMI for men.'''
    w = table[:, 3] / 2.2
    h = table[:, 2] / 3.3
    bmi = w / h ** 2
    m = np.mean(bmi[table[:, 0] == 0])
    return m

def F3(table):
    '''Number of women who are tall, thin, and light'''
    women = table[table[:, 0] == 1]
    tall = women[women[:, 2] > 5.5]
    thin = tall[tall[:, 4] < 26]
    light = thin[thin[:, 3] < 140]
    return len(light)

def F4(table):
    '''Mean and stdev of waist size for men 30 and over'''
    men = table[table[:, 0] == 0]
    old = men[men[:, 1] >= 30]
    waist = old[:, 4]
    mw = np.mean(waist)
    sw = np.std(waist)
    return (mw, sw)

Suppose you have a file like this one except with many more lines. Each line has a alphanumeric code in the first 10 columns and lots of other data after that. You mission is to create a new file containing only the first line with each code. Think of the lines with same code as accidental duplicates that you are trying to remove. Don't change the format of the file in any way, simply omit the lines that duplicate a previous code.

Suppose you trying to make texts more readable to old folks by replacing common acronyms with the phrase they represent. Your acronyms might be represented in a dictionary like this:

    tla = {
    'abt': 'about',
    'adn': 'any day now',
    'irl': 'in real life',
    'lol': 'laugh out loud',
    'b4': 'before'
}

For a text message like 'hello abt to lol' you should produce the output 'hello about to laugh out loud'

For example, the table might look like this:

array([[ 1.  ,  0.47],
       [ 1.  ,  1.07],
       [ 1.  ,  2.22],
       [ 2.  ,  0.79],
       [ 2.  ,  1.61],
       [ 4.  ,  2.18],
       [ 4.  ,  0.98],
       [ 4.  ,  2.14],
       [ 4.  ,  2.21],
       [ 5.  ,  0.56]])

And we want to produce output like this:

1.0 3.76
2.0 2.4
4.0 7.51
5.0 0.56

We looked at two solutions. The first is easiest and uses a dictionary just as we did when counting values. This time we're summing them up rather than counting.

def printSums(table):
    sum = {}  # initialize an empty dictionary
    for row in table:  # visit each row
        code = row[0]  # unpack row
        amt = row[1]   # and amount
        if code not in sum:  # test if we've seen this code
            sum[code] = 0.0  # add it if not
        sum[code] = sum[code] + amt  # accumulate the amounts

    # we need to sort the keys to get them in order
    for code in sorted(sum.keys()):
        print code, sum[code]

Then we looked at a sequential approach that would use less memory but is trickier to get correct.

def printSums2(table):
    sum = 0  # initialize a single accumulator
    current_code = None  # keep track of the code we're summing
    for row in table:  # visit each row
        code = row[0]  # unpack
        amt = row[1]

        # is the code the same as previous?
        if code == current_code:
            sum = sum + amt  # yes, add this one in
        else:
            # no, then report if not the first one
            if current_code is not None:
                print current_code, sum
            sum = amt  # reiniitalize sum
            current_code = code  # reset the code

    # we must print the last one
    print current_code, sum

# IPython log file

cd ~/Dropbox/116
ls
from glob import glob
glob('*.txt')
#[Out]# ['gb.txt', 'babies.txt', '1-1000.txt']
ls MyData
glob('MyData/*.txt')
#[Out]# ['MyData/hoho.txt', 'MyData/bar.txt', 'MyData/gary.txt', 'MyData/foo.txt']
line = 'bird 7.35'
line.split()
#[Out]# ['bird', '7.35']
line = 'bird|7.35'
line.split()
#[Out]# ['bird|7.35']
line.split('|')
#[Out]# ['bird', '7.35']
'foo' == 'bar'
#[Out]# False
from csv import DictReader
reader = DictReader(file('example.csv'))
for row in reader:
    print row

reader = DictReader(file('example.csv'))
row1 = reader.next()
row1
#[Out]# {'Day No': '1',
#[Out]#  'Description': 'lousy day',
#[Out]#  'Rain Amount': '0.0',
#[Out]#  'Temperature': '47.5'}
row1['Day No']
#[Out]# '1'
reader = DictReader(file('example.csv'))
for row in reader:
    print row['Day No'], row['Description']

# IPython log file

cd ~/Dropbox/116

# load our baby data
data = np.loadtxt('babies.txt')

# look at the first few rows to remind ourselves
data[:5]
#[Out]# array([[ 5.34,  0.  ],
#[Out]#        [ 3.1 ,  1.  ],
#[Out]#        [ 7.5 ,  1.  ],
#[Out]#        [ 8.56,  1.  ],
#[Out]#        [ 6.03,  0.  ]])

# then we wrote our BigBabies function.
import numpy as np

def BigBabies(d):
    'select big babies'
    m = np.mean(d[:,0])  # get the mean
    s = np.std(d[:,0])   # get the standard deviation
    return d[d[:,0] > m + s,:]  # get the big ones

b = BigBabies(data)
b[:5]
#[Out]# array([[ 7.5 ,  1.  ],
#[Out]#        [ 8.56,  1.  ],
#[Out]#        [ 8.25,  1.  ],
#[Out]#        [ 7.63,  1.  ],
#[Out]#        [ 7.78,  1.  ]])

# then we expanded our script to get the percentages of big boys and girls
bigones = BigBabies(data)
bigboys = bigones[bigones[:,1] == 0]
biggirls = bigones[bigones[:,1] == 1]

print 'percentage of big babies that are boys', 100.0 * len(bigboys) / len(bigones)
print 'percentage of big babies that are girls', 100.0 * len(biggirls) / len(bigones)

# then we worked on generating the first N primes.
# here is the code we put in nprimes.py
def isPrime(p):
    for i in range(2, p):
        if p % i == 0:
            return False
    return True

def nprimes(N):
    'return a list of the first N primes'
    R = []
    p = 1
    for i in range(N):
        # add a prime to R
        while not isPrime(p):
            p = p + 1
        R.append(p)
        p = p + 1
    return R

# we broke it into two functions to make coding the nprimes function easier

nprimes(5)
#[Out]# [2, 3, 5, 7, 11]

# someone asked why we needed that second p = p + 1 at the end of the loop?
# so we removed it
run nprimes.py
nprimes(5)

# now we can't get beyond 2 because it is prime so we never run the body of the while loop
#[Out]# [2, 2, 2, 2, 2]

Use our faked up baby data for this exercise.

Define a function BigBabies that takes selects out the babies that are more than 1 standard deviation heavier than the mean and returns them in a new array with 2 columns. Use the numpy powertools rather than a loop.

Write a script that compares the percentage of big boy babies to big girl babies. Again, use the array power tools.

# IPython log file

cd Dropbox/116

# first I faked up some noisy data
x = np.random.random_integers(0, 100, 100) / 10.0
y = 50 + 10 * x + x**2 + 5 * np.random.randn(100)

# look at it plotted at points
plot(x, y, '.')
#[Out]# [<matplotlib.lines.Line2D at 0xb0e57ec>]

# just listing the values isn't very illuminating.
x[:5]
#[Out]# array([ 7.2,  1.6,  3.5,  6.7,  4.9])
y[:5]
#[Out]# array([ 173.19489273,   73.59377412,   99.23120207,  159.96533158,
#[Out]#         118.72251928])

# Let's fit a line to the data.
# Our line equation will be of the form y = m * x + b
# where m is the slope and b is the intercept

# We can set this up as a system of N equations
# y[0] = b + m * x[0]
# y[1] = b + m * x[1]
# ...
# y[N-1] = b + m * x[N-1]

# we have tools to solve a matrix equation of the form M * V = Y
# where M is a N by 2 matrix
# and V is a 2 x 1 matrix
# and Y is a N by 1 matrix

# To get our system of equations into this form we need a matrix to evaluate b + m * x
# we can get b simply added by multiplying in by 1 in our matrix

# That matrix is easy to construct with numpy
# I'll transpose it because I want N rows by 2 columns
#  not 2 rows of N columns
M = np.array([np.ones_like(x), x]).T
M.shape
#[Out]# (100, 2)
M[:5]
#[Out]# array([[ 1. ,  7.2],
#[Out]#        [ 1. ,  1.6],
#[Out]#        [ 1. ,  3.5],
#[Out]#        [ 1. ,  6.7],
#[Out]#        [ 1. ,  4.9]])

# numpy includes a function to produce the least-squares solution to a system of
# linear equations. It is careful to maintain the numerical accuracy of the result

R = np.linalg.lstsq(M, y)

# the result includes several values, the first part is what we want
R
#[Out]# (array([ 33.36284698,  19.86799503]),
#[Out]#  array([ 6930.03699749]),
#[Out]#  2,
#[Out]#  array([ 60.43050388,   4.5764834 ]))

# extract the first part of the solution and call it fit
fit = R[0]
fit
#[Out]# array([ 33.36284698,  19.86799503])
# it is saying the intercept is 33.3 and the slope is about 19.9

# now we can get the estimated y value for each x using this equation
ye = np.dot(M, fit)

# we plotted it and it looks pretty good though it curves a bit at the ends
plot(x, ye, 'r.')
#[Out]# [<matplotlib.lines.Line2D at 0xb0f78ac>]

# let's look at the error
residual = y - ye

# in a new figure
figure(2)
#[Out]# <matplotlib.figure.Figure at 0xb0f7d2c>
plot(x, residual, '.')

# we could see that it was pretty curved. Perhaps this function is quadratic
#[Out]# [<matplotlib.lines.Line2D at 0xab5860c>]

# fitting a quadratic is just as easy, we just add a column of x squared
M2 = np.array([np.ones_like(x), x, x**2]).T
M2.shape
#[Out]# (100, 3)

# get the quadratic fit
R2 = np.linalg.lstsq(M2, y)
R2
#[Out]# (array([ 50.66106284,  10.2079896 ,   0.94856269]),
#[Out]#  array([ 2199.3046329]),
#[Out]#  3,
#[Out]#  array([ 464.38897231,   15.38634848,    2.80652423]))
figure(1)
#[Out]# <matplotlib.figure.Figure at 0xb06dd2c>
fit2 = R2[0]

# now we see it is estimating an intercept of 50.7,
#   slope of 10.2
#   and quadratic coefficient of 0.95.
fit2
#[Out]# array([ 50.66106284,  10.2079896 ,   0.94856269])

# we can get the new estimate
ye2 = np.dot(M2, fit2)

# and plot it to see it fits better
plot(x, ye2, 'g.')
#[Out]# [<matplotlib.lines.Line2D at 0xb0f13cc>]

# we can also look at the residual
residual2 = y - ye2
figure(2)
#[Out]# <matplotlib.figure.Figure at 0xb0f7d2c>

# which now looks like noise instead of having obvious structure
plot(x, residual2, 'g.')
#[Out]# [<matplotlib.lines.Line2D at 0xab58eec>]

# note that our linear fit isn't simply a subset of the quadratic
fit
#[Out]# array([ 33.36284698,  19.86799503])
fit2
#[Out]# array([ 50.66106284,  10.2079896 ,   0.94856269])

# we could fit other types of equations by transforming the data using logarithms.

The key is to use the raw_input(prompt) function. It takes a string argument and returns the string that the user typed.

We started with a trivial script that prompts and gets the response.

ans = raw_input('Say what? ')

Then we grew it to prompt the user for words chosen from the list of the 1000 most common words. We read the words and made them into a list with split. Then we repeatedly chose a random word from the list and asked the user to type it. We recorded the prompt and their response in the list answers.

import random

# load up a list of 1000 common words
words = file('1-1000.txt').read().split()

ntrials = 10

answers = []
for i in range(ntrials):
    word = random.choice(words)
    ans = raw_input('Please type ' + word + ': ')
    answers.append((word, ans))

First we enhanced our script to give our user a break after a few prompts.

import random

# load up a list of 1000 common words
words = file('1-1000.txt').read().split()

ntrials = 10

answers = []
for i in range(ntrials):
    word = random.choice(words)
    ans = raw_input('Please type ' + word + ': ')
    answers.append((word, ans))
    if (i % 4 == 3): # give the user a break
        go = raw_input('take a break, type y to continue: ')

Then we enhanced our script to time the user's response. Now we are recording the prompt, answer, and the time in seconds for the response.

import random
import time

# load up a list of 1000 common words
words = file('1-1000.txt').read().split()

ntrials = 10

answers = []
for i in range(ntrials):
    word = random.choice(words)
    tstart = time.time()
    ans = raw_input('Please type ' + word + ': ')
    tstop = time.time()
    answers.append((word, ans, tstop - tstart))
    if (i % 4 == 3):
        go = raw_input('take a break, type y to continue: ')

Then we went a little crazy and jumbled the prompt. That sure increased the challenge.

import random
import time

# load up a list of 1000 common words
words = file('1-1000.txt').read().split()

ntrials = 10

answers = []
for i in range(ntrials):
    word = random.choice(words)
    prompt = list(word)  # bust the string into list of single letters
    random.shuffle(prompt)  # shuffle the letters
    prompt = ''.join(prompt)  # rejoin the letters into a string
    tstart = time.time()
    ans = raw_input('Please type ' + prompt + ': ')
    tstop = time.time()
    answers.append((word, ans, tstop - tstart))
    if (i % 4 == 3):
        go = raw_input('take a break, type y to continue: ')

Finally, we added writing the data into a file as we went so we don't have to worry about losing it.

import random
import time

id = raw_input('Enter use id: ')  # get some name for the file
outfile = file(id + '.txt', 'w')  # create a file for this user's data

# load up a list of 1000 common words
words = file('1-1000.txt').read().split()

ntrials = 10

answers = []
for i in range(ntrials):
    word = random.choice(words)
    prompt = list(word)
    random.shuffle(prompt)
    prompt = ''.join(prompt)
    tstart = time.time()
    ans = raw_input('Please type ' + prompt + ': ')
    tstop = time.time()
    answers.append((word, ans, tstop - tstart))
    print >>outfile, word, ans, tstop - tstart  # write the data to the file
    if (i % 4 == 3):
        go = raw_input('take a break, type y to continue: ')

outfile.close()  # close the file