SharpDownloader 1.0.1

C# Engine to download multiple WebRequests in Paralel with TLP and Concurrent Blocking COllection

Install-Package SharpDownloader -Version 1.0.1
dotnet add package SharpDownloader --version 1.0.1
<PackageReference Include="SharpDownloader" Version="1.0.1" />
For projects that support PackageReference, copy this XML node into the project file to reference the package.
paket add SharpDownloader --version 1.0.1
The NuGet Team does not provide support for this client. Please contact its maintainers for support.

namespace Sample
{
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using SharpDownloader;
using SharpDownloader.Extensions;

class Program
{
    static void Main(string[] args)
    {

        //Used to define the default reporting settings of the Download manager 
        //Number of processors to use
        //Refresh time 
        //Table cell width

        SharpDownloaderSettings _settings = new SharpDownloaderSettings();

        //Defining the base for adding downloading classes
        SharpDownloaderManager downloaders = new SharpDownloaderManager(_settings);

        //In case you have the source code and you want to do testing 
        
        downloaders.Add(new ContentDownloader(_baseLink:"your base uri",Testing:false,_numberOfProcessors: 5));

        downloaders.StartDownloading();


    }
}

class ContentDownloader : Downloader
{

    public ContentDownloader(string _baseLink, bool Testing = false, int _numberOfProcessors = 0) : base(_baseLink, Testing, _numberOfProcessors)
    {
        this.OnBuildRequests += Base_OnBuildRequests;
        this.OnResponseReturned += Base_OnResponseReturned;
        this.OnProxyChange += Base_OnProxyChange;
        this.OnReleaseResources += Base_OnReleaseResources;
    }
    private async Task<int> Base_OnBuildRequests()
    {
        //Get Format
        //foreach (string item in File.ReadAllLines("Links.txt"))
        //{
        //    HttpWebRequest request = (HttpWebRequest)WebRequest.Create(item);
        //    request.Timeout = 20 * 1000 * 60 * 60;
        //    request.Method = "GET";
        //    request.Host = BaseLink.Host;
        //    request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0";
        //    request.Accept = "text/html";
        //    request.Headers.Add(HttpRequestHeader.AcceptLanguage, "en-US,en;q=0.5");
        //    request.KeepAlive = true;
        //    request.Headers.Add(HttpRequestHeader.Cookie, "your cookie");
        //    request.Headers.Add("Upgrade-Insecure-Requests: 1");
        //    request.Referer = BaseLink.ToString();
        //    WebRequests.Add(new DWebRequest(request, null));
        //}

        //Post Format using Parameters
        //Dictionary<string, string> Formdata = new Dictionary<string, string>
        //    {
        //        {"UserID",""},
        //        { "Username",""},
        //        { "CustomeVar",""},
        //    };
        //
        //HttpWebRequest request = (HttpWebRequest)WebRequest.Create(BaseLink.ToString());
        //request.Timeout = 20 * 1000 * 60 * 60;
        //request.Method = "POST";
        //request.Host = BaseLink.Host;
        //request.ContentType = "application/x-www-form-urlencoded";
        //request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0";
        //request.Accept = "text/html";
        //request.Headers.Add(HttpRequestHeader.AcceptLanguage, "en-US,en;q=0.5");
        //request.KeepAlive = true;
        //request.Headers.Add(HttpRequestHeader.Cookie, "your cookie");
        //request.Headers.Add("Upgrade-Insecure-Requests: 1");
        //request.Referer = BaseLink.ToString();
        //
        //
        //var DRequest = new DWebRequest(request, Formdata) { UsingParams = true };
        //
        //
        //WebRequests.Add(DRequest);

        return WebRequests.Count;
    }
    private Task Base_OnResponseReturned(string _baseLink, string content)
    {
        return Task.Run(() =>
        {
            lock (_LockObject)
            {
                //Increase the TotalDownloader Count to show progress
                Interlocked.Increment(ref total_downloaded);
                //Response has been returned from the WebRequest
                Console.WriteLine(content);
            }
        }
        );
    }
    private async Task<int> Base_OnReleaseResources()
    {
        try
        {
            //Release all your Resources
        }
        catch
        {

        }
        return 1;
    }

    private Task<string> Base_OnProxyChange()
    {
        //Method returns a Task<string> becuase it is awaited on the other side
        return Task.Run(() => {

            Resend:
            try
            {
                HttpClient client = new HttpClient();
                var uri = new Uri(BaseLink.ToString());
                Status = "Changing Proxy";

                string Requestquery = "http://yourproxyimplementation.com?yourURl=";
                var response = client.GetStringAsync(Requestquery+$"{uri.Scheme}"+"://"+$"{uri.Host}").Result;
                if (response != "")
                {
                    Status = "Running";
                    return response;
                }
                else
                {
                    goto Resend;
                }

            }
            catch (Exception e)
            {
                Console.WriteLine(e.Message + BaseLink);
                goto Resend;
            }
        });
    }
}

}

namespace Sample
{
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using SharpDownloader;
using SharpDownloader.Extensions;

class Program
{
    static void Main(string[] args)
    {

        //Used to define the default reporting settings of the Download manager 
        //Number of processors to use
        //Refresh time 
        //Table cell width

        SharpDownloaderSettings _settings = new SharpDownloaderSettings();

        //Defining the base for adding downloading classes
        SharpDownloaderManager downloaders = new SharpDownloaderManager(_settings);

        //In case you have the source code and you want to do testing 
        
        downloaders.Add(new ContentDownloader(_baseLink:"your base uri",Testing:false,_numberOfProcessors: 5));

        downloaders.StartDownloading();


    }
}

class ContentDownloader : Downloader
{

    public ContentDownloader(string _baseLink, bool Testing = false, int _numberOfProcessors = 0) : base(_baseLink, Testing, _numberOfProcessors)
    {
        this.OnBuildRequests += Base_OnBuildRequests;
        this.OnResponseReturned += Base_OnResponseReturned;
        this.OnProxyChange += Base_OnProxyChange;
        this.OnReleaseResources += Base_OnReleaseResources;
    }
    private async Task<int> Base_OnBuildRequests()
    {
        //Get Format
        //foreach (string item in File.ReadAllLines("Links.txt"))
        //{
        //    HttpWebRequest request = (HttpWebRequest)WebRequest.Create(item);
        //    request.Timeout = 20 * 1000 * 60 * 60;
        //    request.Method = "GET";
        //    request.Host = BaseLink.Host;
        //    request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0";
        //    request.Accept = "text/html";
        //    request.Headers.Add(HttpRequestHeader.AcceptLanguage, "en-US,en;q=0.5");
        //    request.KeepAlive = true;
        //    request.Headers.Add(HttpRequestHeader.Cookie, "your cookie");
        //    request.Headers.Add("Upgrade-Insecure-Requests: 1");
        //    request.Referer = BaseLink.ToString();
        //    WebRequests.Add(new DWebRequest(request, null));
        //}

        //Post Format using Parameters
        //Dictionary<string, string> Formdata = new Dictionary<string, string>
        //    {
        //        {"UserID",""},
        //        { "Username",""},
        //        { "CustomeVar",""},
        //    };
        //
        //HttpWebRequest request = (HttpWebRequest)WebRequest.Create(BaseLink.ToString());
        //request.Timeout = 20 * 1000 * 60 * 60;
        //request.Method = "POST";
        //request.Host = BaseLink.Host;
        //request.ContentType = "application/x-www-form-urlencoded";
        //request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0";
        //request.Accept = "text/html";
        //request.Headers.Add(HttpRequestHeader.AcceptLanguage, "en-US,en;q=0.5");
        //request.KeepAlive = true;
        //request.Headers.Add(HttpRequestHeader.Cookie, "your cookie");
        //request.Headers.Add("Upgrade-Insecure-Requests: 1");
        //request.Referer = BaseLink.ToString();
        //
        //
        //var DRequest = new DWebRequest(request, Formdata) { UsingParams = true };
        //
        //
        //WebRequests.Add(DRequest);

        return WebRequests.Count;
    }
    private Task Base_OnResponseReturned(string _baseLink, string content)
    {
        return Task.Run(() =>
        {
            lock (_LockObject)
            {
                //Increase the TotalDownloader Count to show progress
                Interlocked.Increment(ref total_downloaded);
                //Response has been returned from the WebRequest
                Console.WriteLine(content);
            }
        }
        );
    }
    private async Task<int> Base_OnReleaseResources()
    {
        try
        {
            //Release all your Resources
        }
        catch
        {

        }
        return 1;
    }

    private Task<string> Base_OnProxyChange()
    {
        //Method returns a Task<string> becuase it is awaited on the other side
        return Task.Run(() => {

            Resend:
            try
            {
                HttpClient client = new HttpClient();
                var uri = new Uri(BaseLink.ToString());
                Status = "Changing Proxy";

                string Requestquery = "http://yourproxyimplementation.com?yourURl=";
                var response = client.GetStringAsync(Requestquery+$"{uri.Scheme}"+"://"+$"{uri.Host}").Result;
                if (response != "")
                {
                    Status = "Running";
                    return response;
                }
                else
                {
                    goto Resend;
                }

            }
            catch (Exception e)
            {
                Console.WriteLine(e.Message + BaseLink);
                goto Resend;
            }
        });
    }
}

}

Release Notes

Inital Project Structure

NuGet packages

This package is not used by any NuGet packages.

GitHub repositories

This package is not used by any popular GitHub repositories.

Version History

Version Downloads Last updated
1.0.1 240 4/19/2019
1.0.0 197 4/19/2019