diff --git a/02-import.rb b/02-import.rb index a32d907..67feeaf 100644 --- a/02-import.rb +++ b/02-import.rb @@ -13,6 +13,7 @@ def import_movies $db.execute "DELETE FROM Movies;" File.new("data/movies.list").each_line do |l| + l = l.force_encoding('iso-8859-1').encode('utf-8') print "." if (i = i + 1) % 5000 == 0; STDOUT.flush if match = title_re.match(l) stmt.execute!(match[1], match[2].to_i) @@ -31,9 +32,10 @@ def import_times stmt = $db.prepare("UPDATE Movies set length=? WHERE title=? AND year=?;") $db.transaction do File.new("data/running-times.list").each_line do |l| + print "." if (i = i + 1) % 5000 == 0; STDOUT.flush - if match = time_re.match(l) + if match = time_re.match(l.force_encoding('iso-8859-1').encode('utf-8')) stmt.execute!(match[3].to_i, match[1], match[2].to_i) end end @@ -51,6 +53,7 @@ def import_budgets stmt = $db.prepare("UPDATE Movies set budget=? WHERE title=? AND year=?;") $db.transaction do File.new("data/business.list").each(dashes) do |l| + l = l.force_encoding('iso-8859-1').encode('utf-8') if match = title_re.match(l.to_s) and bt = budget_re.match(l.to_s) stmt.execute!(bt[1].gsub!(",","").to_i, match[1], match[2].to_i) end @@ -66,6 +69,7 @@ def import_mpaa_ratings stmt = $db.prepare("UPDATE Movies set mpaa_rating=? WHERE title=? AND year=?;") $db.transaction do File.new("data/mpaa-ratings-reasons.list").each(dashes) do |l| + l = l.force_encoding('iso-8859-1').encode('utf-8') if match = title_re.match(l.to_s) and rt = rating_re.match(l.to_s) stmt.execute!(rt[1], match[1], match[2].to_i) end @@ -84,6 +88,7 @@ def import_genres $db.execute "DELETE FROM Genres;" File.new("data/genres.list").each_line do |l| + l = l.force_encoding('iso-8859-1').encode('utf-8') print "." if (i = i + 1) % 1000 == 0; STDOUT.flush if match = genre_re.match(l) stmt.execute!(match[3], match[1], match[2].to_i) @@ -102,6 +107,7 @@ def import_ratings $db.transaction File.new("data/ratings.list").each_line do |l| + l = l.force_encoding('iso-8859-1').encode('utf-8') if match = ratings_re.match(l) rating, votes, outof10, title, year = match[1], match[2], match[3], match[4], match[5] stmt.execute!(votes, outof10, rating, title, year) @@ -111,8 +117,8 @@ def import_ratings end -# puts "Importing movies" -# import_movies +puts "Importing movies" +import_movies puts "Importing times" import_times puts "Importing budgets" diff --git a/03-export.rb b/03-export.rb index 642d938..c8e4da6 100644 --- a/03-export.rb +++ b/03-export.rb @@ -27,21 +27,23 @@ def ratings_breakdown(ratings) File.open("movies.tab", "w") do |out| out << [ - 'title', 'year', 'length', 'budget', + 'id', 'title', 'year', 'length', 'budget', 'rating', 'votes', (1..10).map{|i| "r" + i.to_s}, 'mpaa', $genres_of_interest ].flatten.join("\t") + "\n" db.execute(sql) do |row| puts i if (i = i + 1) % 5000 == 0 - out << [ - row["title"], - row["year"], - row["length"], - row["budget"], - row["imdb_rating"], row["imdb_votes"], ratings_breakdown(row["imdb_rating_votes"]), - row["mpaa_rating"], genres_binary(row['id'], db) + row[0], #id + row[1], #title + row[2], #year + row[4], #length + row[3], #budget + row[5], #imdb_rating + row[6], #imdb_votes + ratings_breakdown(row[7]), #imdb_rating_breakdown + row[8], genres_binary(row[0], db) ].flatten.join("\t") + "\n" rescue nil end end \ No newline at end of file