[HtmlUnit] SVN: [15471] trunk/htmlunit/src
rbri--- via HtmlUnit-develop <[email protected]> Tue, 17 Jul 2018 18:25:36 +0000
| Newsgroups | gmane.comp.java.htmlunit.devel |
|---|---|
| Message-ID | <[email protected]> |
Revision: 15471
http://sourceforge.net/p/htmlunit/code/15471
Author: rbri
Date: 2018-07-17 18:25:30 +0000 (Tue, 17 Jul 2018)
Log Message:
-----------
next step in our endless encoding fight - more tests are needed
Modified Paths:
--------------
trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/WebRequest.java
trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/html/HtmlAnchor.java
trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/html/HtmlLink.java
trunk/htmlunit/src/test/java/com/gargoylesoftware/htmlunit/PrimitiveWebServer.java
trunk/htmlunit/src/test/java/com/gargoylesoftware/htmlunit/WebClient7Test.java
Modified: trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/WebRequest.java
===================================================================
--- trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/WebRequest.java 2018-07-16 19:03:46 UTC (rev 15470)
+++ trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/WebRequest.java 2018-07-17 18:25:30 UTC (rev 15471)
@@ -64,6 +64,7 @@
private Credentials urlCredentials_;
private Credentials credentials_;
private transient Charset charset_ = ISO_8859_1;
+ private transient Charset urlEncodingCharset_;
/* These two are mutually exclusive; additionally, requestBody_ should only be set for POST requests. */
private List<NameValuePair> requestParameters_ = Collections.emptyList();
@@ -431,10 +432,20 @@
}
/**
- * Returns the character set to use to encode the url params.
- * @return the character set to use to encode the url params
+ * @param charset the character set to use for url (param) encoding
*/
+ public void setUrlEncodingCharset(final Charset charset) {
+ urlEncodingCharset_ = charset;
+ }
+
+ /**
+ * @return the character set to use to encode the url (params)
+ */
public Charset getUrlEncodingCharset() {
+ if (urlEncodingCharset_ != null) {
+ return urlEncodingCharset_;
+ }
+
if (HttpMethod.GET == getHttpMethod()
|| HttpMethod.DELETE == getHttpMethod()
|| HttpMethod.HEAD == getHttpMethod()
Modified: trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/html/HtmlAnchor.java
===================================================================
--- trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/html/HtmlAnchor.java 2018-07-16 19:03:46 UTC (rev 15470)
+++ trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/html/HtmlAnchor.java 2018-07-17 18:25:30 UTC (rev 15471)
@@ -170,6 +170,8 @@
final WebRequest webRequest = new WebRequest(url, browser.getHtmlAcceptHeader());
webRequest.setCharset(page.getCharset());
+ // use the page encoding even if this is a GET requests
+ webRequest.setUrlEncodingCharset(page.getCharset());
webRequest.setAdditionalHeader(HttpHeader.REFERER, page.getUrl().toExternalForm());
if (LOG.isDebugEnabled()) {
LOG.debug(
Modified: trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/html/HtmlLink.java
===================================================================
--- trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/html/HtmlLink.java 2018-07-16 19:03:46 UTC (rev 15470)
+++ trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/html/HtmlLink.java 2018-07-17 18:25:30 UTC (rev 15471)
@@ -227,6 +227,8 @@
final URL url = page.getFullyQualifiedUrl(getHrefAttribute());
final WebRequest request = new WebRequest(url);
+ // use the page encoding even if this is a GET requests
+ request.setUrlEncodingCharset(page.getCharset());
request.setAdditionalHeader(HttpHeader.REFERER, page.getUrl().toExternalForm());
Modified: trunk/htmlunit/src/test/java/com/gargoylesoftware/htmlunit/PrimitiveWebServer.java
===================================================================
--- trunk/htmlunit/src/test/java/com/gargoylesoftware/htmlunit/PrimitiveWebServer.java 2018-07-16 19:03:46 UTC (rev 15470)
+++ trunk/htmlunit/src/test/java/com/gargoylesoftware/htmlunit/PrimitiveWebServer.java 2018-07-17 18:25:30 UTC (rev 15471)
@@ -80,14 +80,18 @@
final Socket socket = server_.accept();
final InputStream in = socket.getInputStream();
final CharArrayWriter writer = new CharArrayWriter();
+
+ String requestString = writer.toString();
int i;
+
while ((i = in.read()) != -1) {
writer.append((char) i);
- if (i == '\n' && writer.toString().endsWith("\r\n\r\n")) {
+ requestString = writer.toString();
+
+ if (i == '\n' && requestString.endsWith("\r\n\r\n")) {
break;
}
}
- final String requestString = writer.toString();
final String response;
if (requestString.contains("/favicon.ico")) {
@@ -97,7 +101,7 @@
+ "Connection: Closed\r\n\r\n";
}
else {
- requests_.add(writer.toString());
+ requests_.add(requestString);
try (OutputStream out = socket.getOutputStream()) {
if (first || otherResponse_ == null) {
response = firstResponse_;
Modified: trunk/htmlunit/src/test/java/com/gargoylesoftware/htmlunit/WebClient7Test.java
===================================================================
--- trunk/htmlunit/src/test/java/com/gargoylesoftware/htmlunit/WebClient7Test.java 2018-07-16 19:03:46 UTC (rev 15470)
+++ trunk/htmlunit/src/test/java/com/gargoylesoftware/htmlunit/WebClient7Test.java 2018-07-17 18:25:30 UTC (rev 15471)
@@ -17,6 +17,7 @@
import static com.gargoylesoftware.htmlunit.BrowserRunner.TestedBrowser.IE;
import java.net.URL;
+import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import org.junit.After;
@@ -132,12 +133,15 @@
}
private void testRequestUrlEncoding(final String url) throws Exception {
+ final String html = "<html>"
+ + "<head><title>foo</title></head>"
+ + "<body></body></html>";
+
final String response = "HTTP/1.1 200 OK\r\n"
- + "Content-Length: 58\r\n"
+ + "Content-Length: " + html.length() + "\r\n"
+ "Content-Type: text/html\r\n"
+ "\r\n"
- + "<html><head><title>foo</title></head><body>"
- + "</body></html>";
+ + html;
primitiveWebServer_ = new PrimitiveWebServer(PORT, response);
primitiveWebServer_.start();
@@ -146,9 +150,111 @@
driver.get(new URL(URL_FIRST, url).toString());
String reqUrl = primitiveWebServer_.getRequests().get(0);
- if (reqUrl.contains("/favicon.ico")) {
- reqUrl = primitiveWebServer_.getRequests().get(1);
+ reqUrl = reqUrl.substring(4, reqUrl.indexOf("HTTP/1.1") - 1);
+
+ assertEquals(getExpectedAlerts()[0], reqUrl);
+ }
+
+ /**
+ * @throws Exception if the test fails
+ */
+ @Test
+ @Alerts(DEFAULT = "/test.html?k%C3%B6nig",
+ IE = "/test.html?k\u00c3\u00b6nig")
+ @NotYetImplemented(IE)
+ public void anchorUrlEncodingUTF8Header() throws Exception {
+ urlEncoding(true, "UTF-8");
+ }
+
+ /**
+ * @throws Exception if the test fails
+ */
+ @Test
+ @Alerts(DEFAULT = "/test.html?k%C3%B6nig",
+ IE = "/test.html?k\u00c3\u00b6nig")
+ @NotYetImplemented(IE)
+ public void anchorUrlEncodingUTF8Meta() throws Exception {
+ urlEncoding(false, "UTF-8");
+ }
+
+ /**
+ * @throws Exception if the test fails
+ */
+ @Test
+ @Alerts(DEFAULT = "/test.html?k%F6nig",
+ IE = "/test.html?k\u00f6nig")
+ @NotYetImplemented(IE)
+ public void anchorUrlEncodingISO8859_1Header() throws Exception {
+ urlEncoding(true, "ISO-8859-1");
+ }
+
+ /**
+ * @throws Exception if the test fails
+ */
+ @Test
+ @Alerts(DEFAULT = "/test.html?k%F6nig",
+ IE = "/test.html?k\u00f6nig")
+ @NotYetImplemented(IE)
+ public void anchorUrlEncodingISO8859_1Meta() throws Exception {
+ urlEncoding(false, "ISO-8859-1");
+ }
+
+ /**
+ * @throws Exception if the test fails
+ */
+ @Test
+ @Alerts("/test.html?k?nig")
+ public void anchorUrlEncodingWindows_1251Header() throws Exception {
+ urlEncoding(true, "Windows-1251");
+ }
+ /**
+ * @throws Exception if the test fails
+ */
+ @Test
+ @Alerts("/test.html?k?nig")
+ public void anchorUrlEncodingWindows_1251Meta() throws Exception {
+ urlEncoding(false, "Windows-1251");
+ }
+
+ private void urlEncoding(final boolean header, final String charset) throws Exception {
+ String html = "<html>\n"
+ + "<head><title>foo</title>\n";
+ if (!header) {
+ html += " <meta http-equiv='Content-Type' content='text/html; charset=" + charset + "'>\n";
}
+
+ html += "</head>\n"
+ + "<body>\n"
+ + " <a id='myLink' href='test.html?k\u00F6nig'>Click me</a>\n"
+ + "</body></html>";
+
+ String firstResponse = "HTTP/1.1 200 OK\r\n"
+ + "Content-Length: " + html.length() + "\r\n"
+ + "Content-Type: text/html";
+ if (header) {
+ firstResponse += "; charset=" + charset;
+ }
+ firstResponse += "\r\n\r\n" + html;
+
+ final String html2 = "<html><head></head><body>"
+ + "</body></html>";
+
+ final String secondResponse = "HTTP/1.1 200 OK\r\n"
+ + "Content-Length: " + html2.length() + "\r\n"
+ + "Content-Type: text/html\r\n"
+ + "\r\n"
+ + html2;
+
+ primitiveWebServer_ = new PrimitiveWebServer(PORT, firstResponse, secondResponse);
+ primitiveWebServer_.setCharset(Charset.forName(charset));
+ primitiveWebServer_.start();
+
+ final WebDriver driver = getWebDriver();
+
+ driver.get(URL_FIRST.toString());
+ driver.findElement(By.id("myLink")).click();
+
+ String reqUrl = primitiveWebServer_.getRequests().get(1);
reqUrl = reqUrl.substring(4, reqUrl.indexOf("HTTP/1.1") - 1);
assertEquals(getExpectedAlerts()[0], reqUrl);
@@ -158,16 +264,59 @@
* @throws Exception if the test fails
*/
@Test
- @Alerts(DEFAULT = "/bug.html?k%EF%BF%BDnig",
- IE = "/bug.html?k\u00ef\u00bf\u00bdnig")
+ @Alerts(DEFAULT = "/test.css?k%C3%B6nig",
+ IE = "/test.css?k\u00c3\u00b6nig")
@NotYetImplemented(IE)
- public void linkUrlEncodingUTF8() throws Exception {
+ public void linkUrlEncodingUTF8Header() throws Exception {
final String html = "<html>\n"
+ "<head><title>foo</title>\n"
+ + " <link rel='stylesheet' type='text/css' href='test.css?k\u00F6nig'>"
+ + "</head>\n"
+ + "<body>\n"
+ + "</body></html>";
+
+ final String firstResponse = "HTTP/1.1 200 OK\r\n"
+ + "Content-Length: " + html.length() + "\r\n"
+ + "Content-Type: text/html; charset=UTF-8\r\n"
+ + "\r\n"
+ + html;
+
+ final String css = "p { color: red; }";
+
+ final String secondResponse = "HTTP/1.1 200 OK\r\n"
+ + "Content-Length: " + css.length() + "\r\n"
+ + "Content-Type: text/css\r\n"
+ + "\r\n"
+ + css;
+
+ primitiveWebServer_ = new PrimitiveWebServer(PORT, firstResponse, secondResponse);
+ primitiveWebServer_.setCharset(StandardCharsets.UTF_8);
+ primitiveWebServer_.start();
+
+ final WebDriver driver = getWebDriver();
+
+ driver.get(URL_FIRST.toString());
+
+ String reqUrl = primitiveWebServer_.getRequests().get(1);
+ reqUrl = reqUrl.substring(4, reqUrl.indexOf("HTTP/1.1") - 1);
+
+ assertEquals(getExpectedAlerts()[0], reqUrl);
+ }
+
+ /**
+ * @throws Exception if the test fails
+ */
+ @Test
+ @Alerts(DEFAULT = "/test.css?k%C3%B6nig",
+ IE = "/test.css?k\u00c3\u00b6nig")
+ @NotYetImplemented(IE)
+ public void linkUrlEncodingUTF8Meta() throws Exception {
+ final String html = "<html>\n"
+ + "<head><title>foo</title>\n"
+ " <meta http-equiv='Content-Type' content='text/html; charset=UTF-8'>\n"
+ + " <link rel='stylesheet' type='text/css' href='test.css?k\u00F6nig'>"
+ "</head>\n"
+ "<body>\n"
- + " <a id='myLink' href='bug.html?k\u00F6nig'>Click me</a>\n"
+ "</body></html>";
final String firstResponse = "HTTP/1.1 200 OK\r\n"
@@ -176,14 +325,56 @@
+ "\r\n"
+ html;
- final String html2 = "<html><head><title>foo</title></head><body>"
+ final String css = "p { color: red; }";
+
+ final String secondResponse = "HTTP/1.1 200 OK\r\n"
+ + "Content-Length: " + css.length() + "\r\n"
+ + "Content-Type: text/css\r\n"
+ + "\r\n"
+ + css;
+
+ primitiveWebServer_ = new PrimitiveWebServer(PORT, firstResponse, secondResponse);
+ primitiveWebServer_.setCharset(StandardCharsets.UTF_8);
+ primitiveWebServer_.start();
+
+ final WebDriver driver = getWebDriver();
+
+ driver.get(URL_FIRST.toString());
+
+ String reqUrl = primitiveWebServer_.getRequests().get(1);
+ reqUrl = reqUrl.substring(4, reqUrl.indexOf("HTTP/1.1") - 1);
+
+ assertEquals(getExpectedAlerts()[0], reqUrl);
+ }
+
+ /**
+ * @throws Exception if the test fails
+ */
+ @Test
+ @Alerts(DEFAULT = "/test.css?k%F6nig",
+ IE = "/test.css?k\u00f6nig")
+ @NotYetImplemented(IE)
+ public void linkUrlEncodingISO8859_1Header() throws Exception {
+ final String html = "<html>\n"
+ + "<head><title>foo</title>\n"
+ + " <link rel='stylesheet' type='text/css' href='test.css?k\u00F6nig'>"
+ + "</head>\n"
+ + "<body>\n"
+ "</body></html>";
+ final String firstResponse = "HTTP/1.1 200 OK\r\n"
+ + "Content-Length: " + html.length() + "\r\n"
+ + "Content-Type: text/html; charset=ISO-8859-1\r\n"
+ + "\r\n"
+ + html;
+
+ final String css = "p { color: red; }";
+
final String secondResponse = "HTTP/1.1 200 OK\r\n"
- + "Content-Length: " + html2.length() + "\r\n"
- + "Content-Type: text/html\r\n"
+ + "Content-Length: " + css.length() + "\r\n"
+ + "Content-Type: text/css\r\n"
+ "\r\n"
- + html2;
+ + css;
primitiveWebServer_ = new PrimitiveWebServer(PORT, firstResponse, secondResponse);
primitiveWebServer_.setCharset(StandardCharsets.ISO_8859_1);
@@ -192,7 +383,6 @@
final WebDriver driver = getWebDriver();
driver.get(URL_FIRST.toString());
- driver.findElement(By.id("myLink")).click();
String reqUrl = primitiveWebServer_.getRequests().get(1);
reqUrl = reqUrl.substring(4, reqUrl.indexOf("HTTP/1.1") - 1);
@@ -204,16 +394,16 @@
* @throws Exception if the test fails
*/
@Test
- @Alerts(DEFAULT = "/bug.html?k%F6nig",
- IE = "/bug.html?k\u00f6nig")
+ @Alerts(DEFAULT = "/test.css?k%F6nig",
+ IE = "/test.css?k\u00f6nig")
@NotYetImplemented(IE)
- public void linkUrlEncodingISO8859_1() throws Exception {
+ public void linkUrlEncodingISO8859_1Meta() throws Exception {
final String html = "<html>\n"
+ "<head><title>foo</title>\n"
+ " <meta http-equiv='Content-Type' content='text/html; charset=ISO-8859-1'>\n"
+ + " <link rel='stylesheet' type='text/css' href='test.css?k\u00F6nig'>"
+ "</head>\n"
+ "<body>\n"
- + " <a id='myLink' href='bug.html?k\u00F6nig'>Click me</a>\n"
+ "</body></html>";
final String firstResponse = "HTTP/1.1 200 OK\r\n"
@@ -222,14 +412,15 @@
+ "\r\n"
+ html;
- final String response = "HTTP/1.1 200 OK\r\n"
- + "Content-Length: 2\r\n"
- + "Content-Type: text/html\r\n"
+ final String css = "p { color: red; }";
+
+ final String secondResponse = "HTTP/1.1 200 OK\r\n"
+ + "Content-Length: " + css.length() + "\r\n"
+ + "Content-Type: text/css\r\n"
+ "\r\n"
- + "<html><head><title>foo</title></head><body>"
- + "</body></html>";
+ + css;
- primitiveWebServer_ = new PrimitiveWebServer(PORT, firstResponse, response);
+ primitiveWebServer_ = new PrimitiveWebServer(PORT, firstResponse, secondResponse);
primitiveWebServer_.setCharset(StandardCharsets.ISO_8859_1);
primitiveWebServer_.start();
@@ -236,12 +427,8 @@
final WebDriver driver = getWebDriver();
driver.get(URL_FIRST.toString());
- driver.findElement(By.id("myLink")).click();
String reqUrl = primitiveWebServer_.getRequests().get(1);
- if (reqUrl.contains("/favicon.ico")) {
- reqUrl = primitiveWebServer_.getRequests().get(2);
- }
reqUrl = reqUrl.substring(4, reqUrl.indexOf("HTTP/1.1") - 1);
assertEquals(getExpectedAlerts()[0], reqUrl);
------------------------------------------------------------------------------
Check out the vibrant tech community on one of the world's most
engaging tech sites, Slashdot.org! http://sdm.link/slashdot