Commit 8783c7f9 authored by Alan Ing's avatar Alan Ing
Browse files

initial

parents

Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 2013
VisualStudioVersion = 12.0.30723.0
MinimumVisualStudioVersion = 10.0.40219.1
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MassGetDb", "MassGetDb\MassGetDb.csproj", "{3882F459-99F6-4104-B465-EB84191E639C}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{3882F459-99F6-4104-B465-EB84191E639C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{3882F459-99F6-4104-B465-EB84191E639C}.Debug|Any CPU.Build.0 = Debug|Any CPU
{3882F459-99F6-4104-B465-EB84191E639C}.Release|Any CPU.ActiveCfg = Release|Any CPU
{3882F459-99F6-4104-B465-EB84191E639C}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
EndGlobal
<?xml version="1.0" encoding="utf-8" ?>
<configuration>
<startup>
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5" />
</startup>
</configuration>
\ No newline at end of file
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<ProjectGuid>{3882F459-99F6-4104-B465-EB84191E639C}</ProjectGuid>
<OutputType>Exe</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>MassGetDb</RootNamespace>
<AssemblyName>MassGetDb</AssemblyName>
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<PlatformTarget>AnyCPU</PlatformTarget>
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<PlatformTarget>AnyCPU</PlatformTarget>
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<ItemGroup>
<Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Newtonsoft.Json.6.0.5\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference>
<Reference Include="System" />
<Reference Include="System.Core" />
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Data.DataSetExtensions" />
<Reference Include="Microsoft.CSharp" />
<Reference Include="System.Data" />
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
<Compile Include="Program.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
</ItemGroup>
<ItemGroup>
<None Include="App.config" />
<None Include="packages.config" />
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">
</Target>
<Target Name="AfterBuild">
</Target>
-->
</Project>
\ No newline at end of file
using Newtonsoft.Json.Linq;
using System;
using System.Data;
using System.Data.SqlClient;
using System.Globalization;
using System.IO;
using System.Net;
using System.Text;
namespace MassGetDb
{
class Program
{
static string connectionString = "AAAAA";
static string[] keyz = new string[] {
"AAAA",
"BBB"
};
static int keyz_counter = 0;
static string bearer_token = ""; //current token, keysz[keyz_counter]
static int tweets_request_count, followers_request_count;
static void Main(string[] args)
{
/*
* always listening on the db
* action will take as long as needed,
* but between actions wait 60 secs to pick up new ones
*/
while (true)
{
action();
System.Threading.Thread.Sleep(1000 * 60);
}
}
private static void action()
{
Console.Write("Establishing database connection... ");
SqlConnection myConnection = new SqlConnection(connectionString);
SqlConnection myConnection2 = new SqlConnection(connectionString);
try
{
myConnection.Open();
Console.WriteLine("ok got db connection!");
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
return;
}
SqlCommand myCommand = new SqlCommand("SELECT * FROM redJOHN.dbo.RIVER WHERE status='waiting' and ttl <> 0 ORDER BY created_date", myConnection);
SqlDataReader dr = myCommand.ExecuteReader();
while (dr.Read())
{
string line_id = dr["line_id"].ToString().Trim();
string user = dr["user"].ToString().Trim();
string type = dr["type"].ToString().Trim();
string action = dr["action"].ToString().Trim();
int ttl = Convert.ToInt16(dr["ttl"].ToString());
string status = string.Format("{0} {1} {2} {3}", line_id, user, type, action);
Console.WriteLine(status);
// grab oldest one marked 'waiting' and change status to in progress
myConnection2.Open();
SqlCommand rewrite = new SqlCommand("UPDATE redJOHN.dbo.RIVER SET status = 'in progress' where line_id = " + line_id, myConnection2);
rewrite.ExecuteNonQuery();
// then take those values and decide what to do with them.
process_input_line(user, type, action, ttl);
SqlCommand rewrite2 = new SqlCommand("UPDATE redJOHN.dbo.RIVER SET status = 'completed' where line_id = " + line_id, myConnection2);
rewrite2.ExecuteNonQuery();
myConnection2.Close();
}
try
{
myConnection.Close();
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
return;
}
}
public static void process_input_line(string user, string type, string action, int ttl)
{
string method = "GET";
string host = "api.twitter.com";
string endpoint = "";
if (action.Equals("tweets")) { endpoint = "https://api.twitter.com/1.1/statuses/user_timeline.json"; }
else if (action.Equals("followers")) { endpoint = "https://api.twitter.com/1.1/followers/ids.json"; }
handle_request(method, host, endpoint, user, type, action, ttl);
}
public static void handle_request(string method, string host, string endpoint, string line_user, string type, string action, int ttl)
{
string user = "";
if (type.Equals("ids")) { user = "user_id=" + line_user; }
else if (type.Equals("screennames")) { user = "screen_name=" + line_user; }
string max_id = "";
string count = "count=200";
string cursor = "cursor=-1";
string body = "";
while (true)
{
string query = "";
if (action.Equals("tweets")) { query = count + "&" + user + "&" + max_id; }
else if (action.Equals("followers")) { query = user + "&" + cursor; }
string response = process_request(method, host, endpoint, query, body, action); // pass in action to ensure correct requests remaining
dynamic stuff = null;
if (action.Equals("tweets")) { response = @"{ info: " + response + "}"; } // tweets json structured weirdly (starts with arrays)
// check private account exception
try { stuff = JObject.Parse(response); }
catch (Exception ex) { return; }
//check for no content
int length = 0;
if (action.Equals("tweets")) { length = stuff.info.Count; }
else if (action.Equals("followers")) { length = stuff.ids.Count; }
if (length == 0) { return; }
// dump to db
if (action.Equals("followers"))
{
SqlConnection sc1 = new SqlConnection(connectionString);
try { sc1.Open(); }
catch (Exception) { }
for (int i = 0; i < stuff.ids.Count; i++)
{
SqlCommand sc = sc1.CreateCommand();
string sql = @"INSERT INTO [redJOHN].[dbo].[RIVER]( [created_date] ,[user] ,[type] ,[action] ,[status] ,[ttl])
VALUES( GETDATE() ,'" + (stuff.ids[i]).ToString().Trim() + "' ,'ids' ,'tweets' ,'waiting' ,-1)";
sc.CommandText = sql;
sc.ExecuteNonQuery();
sql = @"INSERT INTO [redJOHN].[dbo].[RIVER]( [created_date] ,[user] ,[type] ,[action] ,[status] ,[ttl])
VALUES( GETDATE() ,'" + (stuff.ids[i]).ToString().Trim() + "','ids' ,'followers' ,'waiting' ," + (ttl - 1) + ")";
sc.CommandText = sql;
sc.ExecuteNonQuery();
}
try { sc1.Close(); }
catch (Exception) { }
//System.IO.File.AppendAllText(save_dir, ((string)stuff.ids[i]).Trim() + Environment.NewLine);
}
else if (action.Equals("tweets"))
{
SqlConnection sc1 = new SqlConnection(connectionString);
try { sc1.Open(); }
catch (Exception) { }
SqlCommand myCommand = null;
//System.IO.File.AppendAllText(save_dir, response + Environment.NewLine);
for (int i = 0; i < stuff["info"].Count; i++)
{
myCommand = new SqlCommand("INSERT INTO TIGER (" +
"contributors, coordinates, created_at, entities, favourite_count, favorited, geo, id_str, in_reply_to_screen_name, in_reply_to_status_id_str, in_reply_to_user_id_str, lang, place, retweet_count, retweeted, source, text, truncated, user_id_str) VALUES " +
"(@contributors, @coordinates, @created_at, @entities, @favourite_count, @favorited, @geo, @id_str, @in_reply_to_screen_name, @in_reply_to_status_id_str, @in_reply_to_user_id_str, @lang, @place, @retweet_count, @retweeted, @source, @text, @truncated, @user_id_str)", sc1);
SqlParameter p1 = new SqlParameter("@contributors", SqlDbType.NChar);
p1.Value = ("" + stuff.info[i].contributors).Trim();
myCommand.Parameters.Add(p1);
SqlParameter p2 = new SqlParameter("@coordinates", SqlDbType.VarChar);
p2.Value = ("" + stuff.info[i].coordinates).Trim();
myCommand.Parameters.Add(p2);
SqlParameter p3 = new SqlParameter("@created_at", SqlDbType.NVarChar);
p3.Value = DateTime.ParseExact(("" + stuff.info[i].created_at).Trim(), "ddd MMM dd HH:mm:ss zzzz yyyy", CultureInfo.InvariantCulture);
myCommand.Parameters.Add(p3);
SqlParameter p4 = new SqlParameter("@entities", SqlDbType.VarChar);
p4.Value = ("" + stuff.info[i].entities).Trim();
myCommand.Parameters.Add(p4);
SqlParameter p5 = new SqlParameter("@favourite_count", SqlDbType.VarChar);
p5.Value = ("" + stuff.info[i].favourite_count).Trim();
myCommand.Parameters.Add(p5);
SqlParameter p6 = new SqlParameter("@favorited", SqlDbType.NChar);
p6.Value = ("" + stuff.info[i].favorited).Trim();
myCommand.Parameters.Add(p6);
SqlParameter p7 = new SqlParameter("@geo", SqlDbType.NChar);
p7.Value = ("" + stuff.info[i].geo).Trim();
myCommand.Parameters.Add(p7);
SqlParameter p8 = new SqlParameter("@id_str", SqlDbType.NChar);
p8.Value = ("" + stuff.info[i].id_str).Trim();
myCommand.Parameters.Add(p8);
SqlParameter p9 = new SqlParameter("@in_reply_to_screen_name", SqlDbType.NChar);
p9.Value = ("" + stuff.info[i].in_reply_to_screen_name).Trim();
myCommand.Parameters.Add(p9);
SqlParameter p10 = new SqlParameter("@in_reply_to_status_id_str", SqlDbType.NChar);
p10.Value = ("" + stuff.info[i].in_reply_to_status_id_str).Trim();
myCommand.Parameters.Add(p10);
SqlParameter p11 = new SqlParameter("@in_reply_to_user_id_str", SqlDbType.NChar);
p11.Value = ("" + stuff.info[i].in_reply_to_user_id_str).Trim();
myCommand.Parameters.Add(p11);
SqlParameter p12 = new SqlParameter("@lang", SqlDbType.NChar);
p12.Value = ("" + stuff.info[i].lang).Trim();
myCommand.Parameters.Add(p12);
SqlParameter p13 = new SqlParameter("@place", SqlDbType.NChar);
p13.Value = ("" + stuff.info[i].place).Trim();
myCommand.Parameters.Add(p13);
SqlParameter p14 = new SqlParameter("@retweet_count", SqlDbType.VarChar);
p14.Value = ("" + stuff.info[i].retweet_count).Trim();
myCommand.Parameters.Add(p14);
SqlParameter p15 = new SqlParameter("@retweeted", SqlDbType.NChar);
p15.Value = ("" + stuff.info[i].retweeted).Trim();
myCommand.Parameters.Add(p15);
SqlParameter p16 = new SqlParameter("@source", SqlDbType.NVarChar);
p16.Value = ("" + stuff.info[i].source).Trim();
myCommand.Parameters.Add(p16);
SqlParameter p17 = new SqlParameter("@text", SqlDbType.NVarChar);
p17.Value = ("" + stuff.info[i].text).Trim();
myCommand.Parameters.Add(p17);
SqlParameter p18 = new SqlParameter("@truncated", SqlDbType.NChar);
p18.Value = ("" + stuff.info[i].truncated).Trim();
myCommand.Parameters.Add(p18);
SqlParameter p19 = new SqlParameter("@user_id_str", SqlDbType.NChar);
p19.Value = ("" + stuff.info[i].user.id_str).Trim();
myCommand.Parameters.Add(p19);
myCommand.CommandTimeout = 36000;
myCommand.ExecuteNonQuery();
}
try { sc1.Close(); }
catch (Exception) { }
}
// then see if there is more
// only check in the case of orders
// tweets will return response.Equals("[]") which is checked for above already
if (action.Equals("followers"))
{
if (((string)stuff.next_cursor_str).Equals("0"))
{
return;
}
}
// looks like we're going to keep on going
// get new parameters
if (action.Equals("tweets"))
{
max_id = "max_id=" + (stuff.info[stuff.info.Count - 1].id - 1);
}
else if (action.Equals("followers"))
{
cursor = "cursor=" + stuff.next_cursor_str;
}
}
}
public static string process_request(string method, string host, string uri, string query, string body, string action)
{
//manage authorization_header (keys)
//manages request count (static) and rate limiting (rate limiting depends on endpoint)
//keyz
//either get a new key, or wait for one to work
//array of all keys
//circle
//get an index of where current key is,
//iterate through array (like a circle)
//check using api if possible to use (enough requests free?)
//if iterated through all array and can't find possible one,
//wait 15 minutes
//try again using same key
if (action.Equals("tweets"))
{
if (tweets_request_count == -1) // intial!
{
keyz_counter = 0;
bearer_token = keyz[keyz_counter];
tweets_request_count = get_request_remaining(action);
}
if (tweets_request_count < 2)
{
//refresh key
keyz_counter++;
if (keyz_counter == keyz.Length)
{
//wait 15 mins
keyz_counter = 0;
System.Threading.Thread.Sleep(900000);
}
bearer_token = keyz[keyz_counter];
tweets_request_count = get_request_remaining(action);
}
}
if (action.Equals("followers"))
{
if (followers_request_count == -1) // intial!
{
keyz_counter = 0;
bearer_token = keyz[keyz_counter];
followers_request_count = get_request_remaining(action);
}
if (followers_request_count < 2)
{
//refresh key
keyz_counter++;
if (keyz_counter == keyz.Length)
{
//wait 15 mins
keyz_counter = 0;
System.Threading.Thread.Sleep(900000);
}
bearer_token = keyz[keyz_counter];
followers_request_count = get_request_remaining(action);
}
}
/*
* we need a global (static) int variable - count
* initialized at -1
* if at -1 then it knows to refresh the count.
* depends of uri/endpoint used. can start at 15 or 300.
* then this count is decremented for every request.
*
* wait till get threashold
* then do what? wait? or swap to new one.
*/
//then do request
//bearer_token = "AAAAAAAAAAAAAAAAAAAAAPZnUAAAAAAAvS8voydSdwxR2kSrI80TP7vvvb8%3DkYB1GtNKp3Z8ysg6abZLb7QnV7MCPZ1DnGMeMPM5ECdh6dtQlM";
string authorization_header = "Bearer " + bearer_token;
if (action.Equals("followers"))
{
followers_request_count--;
}
else
{
tweets_request_count--;
}
return do_request(method, host, uri, query, body, authorization_header);
}
public static string do_request(string method, string host, string uri, string query, string body, string authorization_header)
{
HttpWebRequest http_request = null;
Stream dataStream = null;
HttpWebResponse http_response = null;
StreamReader reader = null;
if (method.Equals("GET"))
{
http_request = (HttpWebRequest)WebRequest.Create(uri + "?" + query);
}
else if (method.Equals("POST"))
{
http_request = (HttpWebRequest)WebRequest.Create(uri);
}
else
{
return "Error!";
}
http_request.Method = method;
http_request.Host = host;
http_request.UserAgent = "Alan's Test Application";
http_request.Headers.Add("Authorization", authorization_header);
if (method.Equals("POST"))
{
byte[] bodydata = Encoding.UTF8.GetBytes(body);
http_request.ContentType = "application/x-www-form-urlencoded;charset=UTF-8";
http_request.ContentLength = bodydata.Length;
dataStream = http_request.GetRequestStream();
dataStream.Write(bodydata, 0, bodydata.Length);
dataStream.Close();
}
string responseFromServer = "";
try
{
http_response = (HttpWebResponse)http_request.GetResponse();
dataStream = http_response.GetResponseStream();
reader = new StreamReader(dataStream);
responseFromServer = reader.ReadToEnd();
}
catch (Exception e2)
{
responseFromServer = e2.ToString();
}
finally
{
// Cleanup the streams and the response.
if (reader != null) { reader.Close(); }
if (http_response != null) { http_response.Close(); }
if (dataStream != null) { dataStream.Close(); }
}
return responseFromServer;
}
public static int get_request_remaining(string action)
{
int request_count = -2;
string response2 = do_request("GET", "api.twitter.com", "https://api.twitter.com/1.1/application/rate_limit_status.json", "resources=application,followers,statuses,users", "", "Bearer " + bearer_token);
dynamic stuff = null;
try
{
stuff = JObject.Parse(response2);
}
catch (Exception ex)
{
return 1;
}
if (action.Equals("tweets"))
{
request_count = int.Parse((string)stuff.resources.statuses["/statuses/user_timeline"].remaining);
}
else if (action.Equals("followers"))
{
request_count = int.Parse((string)stuff.resources.followers["/followers/ids"].remaining);
}
return request_count;
}
}
}
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("MassGetDb")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("Microsoft")]
[assembly: AssemblyProduct("MassGetDb")]
[assembly: AssemblyCopyright("Copyright © Microsoft 2014")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]
// The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("0d4ee486-e595-4dc7-bd0d-5c13485cb210")]
// Version information for an assembly consists of the following four values:
//
// Major Version
// Minor Version
// Build Number
// Revision
//
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.0.0.0")]
[assembly: AssemblyFileVersion("1.0.0.0")]
<?xml version="1.0" encoding="utf-8" ?>
<configuration>
<startup>
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5" />
</startup>
</configuration>
\ No newline at end of file
<?xml version="1.0" encoding="utf-8" ?>
<configuration>
<startup>
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5" />
</startup>
</configuration>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<assembly xmlns="urn:schemas-microsoft-com:asm.v1" manifestVersion="1.0">
<assemblyIdentity version="1.0.0.0" name="MyApplication.app"/>
<trustInfo xmlns="urn:schemas-microsoft-com:asm.v2">
<security>
<requestedPrivileges xmlns="urn:schemas-microsoft-com:asm.v3">
<requestedExecutionLevel level="asInvoker" uiAccess="false"/>
</requestedPrivileges>
</security>
</trustInfo>
</assembly>