SharpDownloader 1.0.1

C# Engine to download multiple WebRequests in Paralel with TLP and Concurrent Blocking COllection

Install-Package SharpDownloader -Version 1.0.1
dotnet add package SharpDownloader --version 1.0.1
<PackageReference Include="SharpDownloader" Version="1.0.1" />
For projects that support PackageReference, copy this XML node into the project file to reference the package.
paket add SharpDownloader --version 1.0.1
The NuGet Team does not provide support for this client. Please contact its maintainers for support.

namespace Sample
{
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using SharpDownloader;
using SharpDownloader.Extensions;

class Program
{
    static void Main(string[] args)
    {

        //Used to define the default reporting settings of the Download manager 
        //Number of processors to use
        //Refresh time 
        //Table cell width

        SharpDownloaderSettings _settings = new SharpDownloaderSettings();

        //Defining the base for adding downloading classes
        SharpDownloaderManager downloaders = new SharpDownloaderManager(_settings);

        //In case you have the source code and you want to do testing 
        
        downloaders.Add(new ContentDownloader(_baseLink:"your base uri",Testing:false,_numberOfProcessors: 5));

        downloaders.StartDownloading();


    }
}

class ContentDownloader : Downloader
{

    public ContentDownloader(string _baseLink, bool Testing = false, int _numberOfProcessors = 0) : base(_baseLink, Testing, _numberOfProcessors)
    {
        this.OnBuildRequests += Base_OnBuildRequests;
        this.OnResponseReturned += Base_OnResponseReturned;
        this.OnProxyChange += Base_OnProxyChange;
        this.OnReleaseResources += Base_OnReleaseResources;
    }
    private async Task<int> Base_OnBuildRequests()
    {
        //Get Format
        //foreach (string item in File.ReadAllLines("Links.txt"))
        //{
        //    HttpWebRequest request = (HttpWebRequest)WebRequest.Create(item);
        //    request.Timeout = 20 * 1000 * 60 * 60;
        //    request.Method = "GET";
        //    request.Host = BaseLink.Host;
        //    request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0";
        //    request.Accept = "text/html";
        //    request.Headers.Add(HttpRequestHeader.AcceptLanguage, "en-US,en;q=0.5");
        //    request.KeepAlive = true;
        //    request.Headers.Add(HttpRequestHeader.Cookie, "your cookie");
        //    request.Headers.Add("Upgrade-Insecure-Requests: 1");
        //    request.Referer = BaseLink.ToString();
        //    WebRequests.Add(new DWebRequest(request, null));
        //}

        //Post Format using Parameters
        //Dictionary<string, string> Formdata = new Dictionary<string, string>
        //    {
        //        {"UserID",""},
        //        { "Username",""},
        //        { "CustomeVar",""},
        //    };
        //
        //HttpWebRequest request = (HttpWebRequest)WebRequest.Create(BaseLink.ToString());
        //request.Timeout = 20 * 1000 * 60 * 60;
        //request.Method = "POST";
        //request.Host = BaseLink.Host;
        //request.ContentType = "application/x-www-form-urlencoded";
        //request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0";
        //request.Accept = "text/html";
        //request.Headers.Add(HttpRequestHeader.AcceptLanguage, "en-US,en;q=0.5");
        //request.KeepAlive = true;
        //request.Headers.Add(HttpRequestHeader.Cookie, "your cookie");
        //request.Headers.Add("Upgrade-Insecure-Requests: 1");
        //request.Referer = BaseLink.ToString();
        //
        //
        //var DRequest = new DWebRequest(request, Formdata) { UsingParams = true };
        //
        //
        //WebRequests.Add(DRequest);

        return WebRequests.Count;
    }
    private Task Base_OnResponseReturned(string _baseLink, string content)
    {
        return Task.Run(() =>
        {
            lock (_LockObject)
            {
                //Increase the TotalDownloader Count to show progress
                Interlocked.Increment(ref total_downloaded);
                //Response has been returned from the WebRequest
                Console.WriteLine(content);
            }
        }
        );
    }
    private async Task<int> Base_OnReleaseResources()
    {
        try
        {
            //Release all your Resources
        }
        catch
        {

        }
        return 1;
    }

    private Task<string> Base_OnProxyChange()
    {
        //Method returns a Task<string> becuase it is awaited on the other side
        return Task.Run(() => {

            Resend:
            try
            {
                HttpClient client = new HttpClient();
                var uri = new Uri(BaseLink.ToString());
                Status = "Changing Proxy";

                string Requestquery = "http://yourproxyimplementation.com?yourURl=";
                var response = client.GetStringAsync(Requestquery+$"{uri.Scheme}"+"://"+$"{uri.Host}").Result;
                if (response != "")
                {
                    Status = "Running";
                    return response;
                }
                else
                {
                    goto Resend;
                }

            }
            catch (Exception e)
            {
                Console.WriteLine(e.Message + BaseLink);
                goto Resend;
            }
        });
    }
}

}

namespace Sample
{
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using SharpDownloader;
using SharpDownloader.Extensions;

class Program
{
    static void Main(string[] args)
    {

        //Used to define the default reporting settings of the Download manager 
        //Number of processors to use
        //Refresh time 
        //Table cell width

        SharpDownloaderSettings _settings = new SharpDownloaderSettings();

        //Defining the base for adding downloading classes
        SharpDownloaderManager downloaders = new SharpDownloaderManager(_settings);

        //In case you have the source code and you want to do testing 
        
        downloaders.Add(new ContentDownloader(_baseLink:"your base uri",Testing:false,_numberOfProcessors: 5));

        downloaders.StartDownloading();


    }
}

class ContentDownloader : Downloader
{

    public ContentDownloader(string _baseLink, bool Testing = false, int _numberOfProcessors = 0) : base(_baseLink, Testing, _numberOfProcessors)
    {
        this.OnBuildRequests += Base_OnBuildRequests;
        this.OnResponseReturned += Base_OnResponseReturned;
        this.OnProxyChange += Base_OnProxyChange;
        this.OnReleaseResources += Base_OnReleaseResources;
    }
    private async Task<int> Base_OnBuildRequests()
    {
        //Get Format
        //foreach (string item in File.ReadAllLines("Links.txt"))
        //{
        //    HttpWebRequest request = (HttpWebRequest)WebRequest.Create(item);
        //    request.Timeout = 20 * 1000 * 60 * 60;
        //    request.Method = "GET";
        //    request.Host = BaseLink.Host;
        //    request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0";
        //    request.Accept = "text/html";
        //    request.Headers.Add(HttpRequestHeader.AcceptLanguage, "en-US,en;q=0.5");
        //    request.KeepAlive = true;
        //    request.Headers.Add(HttpRequestHeader.Cookie, "your cookie");
        //    request.Headers.Add("Upgrade-Insecure-Requests: 1");
        //    request.Referer = BaseLink.ToString();
        //    WebRequests.Add(new DWebRequest(request, null));
        //}

        //Post Format using Parameters
        //Dictionary<string, string> Formdata = new Dictionary<string, string>
        //    {
        //        {"UserID",""},
        //        { "Username",""},
        //        { "CustomeVar",""},
        //    };
        //
        //HttpWebRequest request = (HttpWebRequest)WebRequest.Create(BaseLink.ToString());
        //request.Timeout = 20 * 1000 * 60 * 60;
        //request.Method = "POST";
        //request.Host = BaseLink.Host;
        //request.ContentType = "application/x-www-form-urlencoded";
        //request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0";
        //request.Accept = "text/html";
        //request.Headers.Add(HttpRequestHeader.AcceptLanguage, "en-US,en;q=0.5");
        //request.KeepAlive = true;
        //request.Headers.Add(HttpRequestHeader.Cookie, "your cookie");
        //request.Headers.Add("Upgrade-Insecure-Requests: 1");
        //request.Referer = BaseLink.ToString();
        //
        //
        //var DRequest = new DWebRequest(request, Formdata) { UsingParams = true };
        //
        //
        //WebRequests.Add(DRequest);

        return WebRequests.Count;
    }
    private Task Base_OnResponseReturned(string _baseLink, string content)
    {
        return Task.Run(() =>
        {
            lock (_LockObject)
            {
                //Increase the TotalDownloader Count to show progress
                Interlocked.Increment(ref total_downloaded);
                //Response has been returned from the WebRequest
                Console.WriteLine(content);
            }
        }
        );
    }
    private async Task<int> Base_OnReleaseResources()
    {
        try
        {
            //Release all your Resources
        }
        catch
        {

        }
        return 1;
    }

    private Task<string> Base_OnProxyChange()
    {
        //Method returns a Task<string> becuase it is awaited on the other side
        return Task.Run(() => {

            Resend:
            try
            {
                HttpClient client = new HttpClient();
                var uri = new Uri(BaseLink.ToString());
                Status = "Changing Proxy";

                string Requestquery = "http://yourproxyimplementation.com?yourURl=";
                var response = client.GetStringAsync(Requestquery+$"{uri.Scheme}"+"://"+$"{uri.Host}").Result;
                if (response != "")
                {
                    Status = "Running";
                    return response;
                }
                else
                {
                    goto Resend;
                }

            }
            catch (Exception e)
            {
                Console.WriteLine(e.Message + BaseLink);
                goto Resend;
            }
        });
    }
}

}

Release Notes

Inital Project Structure

This package is not used by any popular GitHub repositories.

Version History

Version Downloads Last updated
1.0.1 135 4/19/2019
1.0.0 113 4/19/2019