salimsen/emmytweets

Analyses tweets from uganda for a particular period of time # emmytweets Analyses tweets from uganda for a particular period of time (require data-science-master) (require plot) (require math) (require json) (require srfi/19) (require racket/stream) ;;; This function reads line-oriented JSON (as output by massmine), ;;; and packages it into an array. For very large data sets, loading ;;; everything into memory like this is heavy handed. For data this small, ;;; working in memory is simpler (define (json-lines->json-array #:head [head #f]) (let loop ([num 0] [json-array '()] [record (read-json (current-input-port))]) (if (or (eof-object? record) (and head (>= num head))) (jsexpr->string json-array) (loop (add1 num) (cons record json-array) (read-json (current-input-port)))))) ;;; Normalize case, remove URLs, remove punctuation, and remove spaces ;;; from each tweet. This function takes a list of words and returns a ;;; preprocessed subset of words/tokens as a list (define (preprocess-text lst) (map (λ (x) (string-normalize-spaces (remove-punctuation (remove-urls (string-downcase x))) #:websafe? #t)) lst)) ;;; Read in the entire tweet database (3200 tweets from Trump's timeline) (define tweets (string->jsexpr (with-input-from-file "TweetsUganda400.json" (λ () (json-lines->json-array))))) ;; Remove just the tweet text and source from each tweet ;;; hash. Finally, remove retweets. ;;; Remove just the tweet text, source, and timestamp from each tweet ;;; hash. Finally, remove retweets. (define t (let ([tmp (map (λ (x) (list (hash-ref x 'text))) tweets)]) ;; improve to use streams (filter (λ (x) (not (string-prefix? (first x) "RT"))) tmp) )) ;;t is a list of lists of strings. Tail recursion is used to extract each string and append ;; it into one large string. (define joined-tweets (local[ (define (joined1 tlist1 acc) (cond [(empty? tlist1) acc] [else (joined1 (rest tlist1) (string-join (list acc "\n " (first(first tlist1)))))] ) ) ](joined1 t "")) ) ;;; To begin our sentiment analysis, we extract each unique word ;; …