web form filling emulation

[Merrion]

I have written an app that "fills in" a web form and posts it.

This is done by encoding the parameters in multipart/form-data mode e.g.:-

Public Class multipart_formdata

#Region "Private constants"
   Private Const DEF_BOUNDARY As String = "-----------------------------7d6386366801f6"
#End Region

#Region "Private members"
   Private _Boundary As String
   Private _FormText As New StringBuilder
#End Region

#Region "Public interface"
   Public ReadOnly Property MultipartBoundary() As String
       Get
           Return _Boundary
       End Get
   End Property

   Public ReadOnly Property Text() As String
       Get
           Return _FormText.ToString & " " & _Boundary & "--  "
       End Get
   End Property

   Public Sub AddParameter(ByVal ParameterName As String, ByVal ParameterValue As String)

       _FormText.Append(_Boundary) '----------------------------7ce3023980c 
       _FormText.Append(" ")
       _FormText.Append("Content-Disposition: form-data; name=")
       _FormText.Append(QuoteString(ParameterName))
       _FormText.Append(" ")
       _FormText.Append(ParameterValue)
       _FormText.Append(" ")
       'Content-Disposition: form-data; name="UploadFormName"; filename="C:\Directory\UploadFile.txt" 
       'Content-Type: text/plain '

       'This is a test file for ProjectUpload.
       '-----------------------------7ce3023980c-- 

   End Sub
#End Region

#Region "Public constructor"
   Public Sub New()
       Me.New(DEF_BOUNDARY)
   End Sub

   Public Sub New(ByVal Boundary As String)

       If Boundary Is Nothing OrElse Boundary = "" Then
           _Boundary = DEF_BOUNDARY
       Else
           _Boundary = Boundary
       End If

   End Sub
#End Region

#Region "Private methods"
   Private Function QuoteString(ByVal sIn As String) As String
       Return Chr(34) & sIn & Chr(34)
   End Function
#End Region

End Class

 

Then , for example to send a usename and password to a URL you could do thus:-

           Dim webReq As HttpWebRequest = CType(WebRequest.Create("http://www.test.com"), HttpWebRequest)
           Dim byteOut() As Byte
           Dim _form As New multipart_formdata
           With _form
               .AddParameter("username", _Username)
               .AddParameter("password", _Password)
           End With

           Dim encoder As New System.Text.UTF8Encoding
           byteOut = encoder.GetBytes(_form.Text)
           With webReq
               .CachePolicy = New System.Net.Cache.RequestCachePolicy(System.Net.Cache.RequestCacheLevel.BypassCache)
               .Accept = "*/*"
               .Method = "POST" 'set the request method to POST
               .ContentType = "multipart/form-data; boundary=" & _form.MultipartBoundary
               .ContentLength = byteOut.Length - 1
               .Expect = Nothing
               .Referer = "http://localhost/PresentValueServlet/getValuation.htm"
           End With
           Dim outStream As Stream
           outStream = webReq.GetRequestStream()
           If outStream.CanWrite Then
                       '\\ write the request string to it...
                       outStream.Write(byteOut, 0, byteOut.Length - 1)
                       outStream.Close()
            End If

 

The problem is that even though the form data is sent to the web service, the Page.Form property is blank at the recieving end - any ideas?

[Gill Bates]

Unless you plan to upload a file I would not use "multipart/form-data" for the content type. You should use "application/x-www-form-urlencoded"" instead. This is the way I usually do it:

StringBuilder builder = new StringBuilder();
builder.Append("name=");
builder.Append(System.Web.HttpUtility.UrlEncode("Gill Bates"));
builder.Append("&country=");
builder.Append(System.Web.HttpUtility.UrlEncode("United States"));

byte[] data = Encoding.ASCII.GetBytes(builder.ToString());

HttpWebRequest request = (HttpWebRequest)WebRequest.Create(new Uri("http://localhost:4087/post_test/form.aspx"));
request.Method = "POST";
request.ContentType = "application/x-www-form-urlencoded";
request.ContentLength = data.Length;

Stream stream = request.GetRequestStream();
stream.Write(data, 0, data.Length);
stream.Close();

WebResponse response = request.GetResponse();
StreamReader reader = new StreamReader(response.GetResponseStream());
Console.WriteLine(reader.ReadToEnd());
reader.Close();
response.GetResponseStream().Close();

[Merrion]

Unfortunately I do need to upload a file - the web site that I'm automating takes a username, password and file to upload...

 

I have written an ASP page to emulate it thus:-

   Protected Sub Page_Load(ByVal sender As Object, ByVal e As System.EventArgs) Handles Me.Load

       Dim RequestOK As Boolean
       Dim input(Request.TotalBytes) As Byte
       Request.InputStream.Read(input, 0, Request.TotalBytes)

       'Don't send a response until the input is processed...
       Page.Response.BufferOutput = True
       Page.Response.ContentType = "application/xml"

       With Request.Form
           If .Item("username") = "Merrion" Then
               If .Item("password") = "Duncan" Then
                       If Page.Request.Files.Count > 0 Then
                           RequestOK = True
                           Page.Request.Files(0).SaveAs("c:\temp\uploaded_file.xml")
                       End If
               End If
           End If
           If Not RequestOK Then
               Dim ErrOutput As New XmlTextWriter(Page.Response.OutputStream, System.Text.Encoding.UTF8)
               With ErrOutput
                   .WriteStartDocument()
                   .WriteStartElement("Return")
                   .WriteStartElement("Settings")
                   .WriteElementString("Method", Request.HttpMethod)
                   For h As Integer = 0 To Request.Headers.Count - 1
                       .WriteElementString(Request.Headers.Keys(h), Request.Headers.Item(h))
                   Next
                   .WriteEndElement()
                   .WriteStartElement("DataContent")
                   If input.Length = 0 Then
                       .WriteElementString("data", "Not set")
                   Else
                       Dim foo As New System.Text.ASCIIEncoding
                       .WriteElementString("data", foo.GetChars(input, 0, input.Length))
                   End If
                   .WriteEndElement()
                   .WriteStartElement("Errors")
                   .WriteElementString("FormValues", Request.Form.Count.ToString)
                   For n As Integer = 0 To Request.Form.Count - 1
                       .WriteElementString(Request.Form.Keys.Item(n), Request.Form.Item(n))
                   Next
                   .WriteEndElement() 'Errors
                   .WriteEndElement() 'Return 
                   .WriteEndDocument()
                   .Close()
               End With
           End If
       End With


   End Sub

 

But when I write to it from my app (as per the code above) it has no values in the Page.Forms collection - i.e. the return is:-

<?xml version="1.0" encoding="utf-8"?>
<Return>
<Settings>
<Method>POST</Method>
<Connection>Keep-Alive</Connection>
<Content-Length>364</Content-Length>
<Content-Type>multipart/form-data; boundary=-----------------------------7d6386366801f6</Content-Type>
<Accept>application/xml</Accept>
<Expect>100-continue</Expect>
<Host>localhost</Host>
</Settings>
<DataContent>
<data>-----------------------------7d6386366801f6
Content-Disposition: form-data; name="username"

Merrion
-----------------------------7d6386366801f6
Content-Disposition: form-data; name="password"

testpwd
-----------------------------7d6386366801f6
Content-Disposition: form-data; name="asof"

04Jul06
-----------------------------7d6386366801f6--

</data>
</DataContent>
<Errors>
<FormValues>0</FormValues>
</Errors>
</Return>

 

but when I use an HTML form to send the data I get:-

<?xml version="1.0" encoding="utf-8" ?> 
- <Return>
- <Settings>
 <Method>POST</Method> 
 <Cache-Control>no-cache</Cache-Control> 
 <Connection>Keep-Alive</Connection> 
 <Content-Length>350</Content-Length> 
 <Content-Type>multipart/form-data; boundary=---------------------------7d61021b7503f8</Content-Type> 
 <Accept>*/*</Accept> 
 <Accept-Encoding>gzip, deflate</Accept-Encoding> 
 <Accept-Language>en-ie</Accept-Language> 
 <Host>localhost</Host> 
 <Referer>http://localhost/PresentValueServlet/getValuation.htm</Referer> 
 <User-Agent>Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 1.0.3705; .NET CLR 2.0.50727)</User-Agent> 
 </Settings>
- <DataContent>
 <data>-----------------------------7d61021b7503f8 
Content-Disposition: form-data; name="username" 

Merrion 
-----------------------------7d61021b7503f8 
Content-Disposition: form-data; name="password"

Duncan 
-----------------------------7d61021b7503f8 
Content-Disposition: form-data; name="asof" 
-----------------------------7d61021b7503f8-- 
</data> 
 </DataContent>
- <Errors>
 <FormValues>3</FormValues> 
 <username>Merrion</username> 
 <password>Duncan</password> 
 <asof /> 
 </Errors>
 </Return>