Java豆瓣电影爬虫 详解Java豆瓣电影爬虫——小爬虫成长记（附源码）

<dependency>

  <groupId>org.springframework</groupId>

  <artifactId>spring-jdbc</artifactId>

  <version>4.1.1.RELEASE</version>

</dependency>

<dependency>

  <groupId>commons-pool</groupId>

  <artifactId>commons-pool</artifactId>

  <version>1.6</version>

</dependency>

<dependency>

  <groupId>commons-dbcp</groupId>

  <artifactId>commons-dbcp</artifactId>

  <version>1.4</version>

</dependency>

<dependency>

  <groupId>mysql</groupId>

  <artifactId>mysql-connector-java</artifactId>

  <version>5.1.38</version>

</dependency>

<dependency>

  <groupId>mysql</groupId>

  <artifactId>mysql-connector-java</artifactId>

  <version>5.1.38</version>

</dependency> 

 <context:property-placeholder location="classpath*:*.properties"/>

<bean id="dataSource" class="org.apache.commons.dbcp.BasicDataSource" destroy-method="close">

  <property name="driverClassName" value="${jdbc.driver}"/>

  <property name="url" value="${jdbc.url}"/>

  <property name="username" value="${jdbc.username}"/>

  <property name="password" value="${jdbc.password}"/>

</bean> 

<dependency>

  <groupId>org.htmlparser</groupId>

  <artifactId>htmlparser</artifactId>

  <version>1.6</version>

</dependency> 

<dependency>

  <groupId>org.apache.httpcomponents</groupId>

  <artifactId>httpclient</artifactId>

  <version>4.5.2</version>

</dependency>　

<mirrors>

  <mirror>

   <id>alimaven</id>

   <name>aliyun maven</name>

   <url>http://maven.aliyun.com/nexus/content/groups/public/</url>

   <mirrorOf>central</mirrorOf>    

  </mirror>

</mirrors> 

@Test

  public void testFile(){

    File seedFile = new File(this.getClass().getResource("/seed.properties").getPath());

    System.out.print("===========" + seedFile.length() + "===========" );

  }

package java.util.regex;

import java.util.Objects;

public final class Matcher implements MatchResult {

  /**

   * The Pattern object that created this Matcher.

   */

  Pattern parentPattern;

 

  /**

   * The storage used by groups. They may contain invalid values if

   * a group was skipped during the matching.

   */

  int[] groups;

 

  /**

   * The range within the sequence that is to be matched. Anchors

   * will match at these "hard" boundaries. Changing the region

   * changes these values.

   */

  int from, to;

 

  /**

   * Lookbehind uses this value to ensure that the subexpression

   * match ends at the point where the lookbehind was encountered.

   */

  int lookbehindTo;

 

  /**

   * The original string being matched.

   */

  CharSequence text;

 

  /**

   * Matcher state used by the last node. NOANCHOR is used when a

   * match does not have to consume all of the input. ENDANCHOR is

   * the mode used for matching all the input.

   */

  static final int ENDANCHOR = 1;

  static final int NOANCHOR = 0;

  int acceptMode = NOANCHOR;

 

  /**

   * The range of string that last matched the pattern. If the last

   * match failed then first is -1; last initially holds 0 then it

   * holds the index of the end of the last match (which is where the

   * next search starts).

   */

  int first = -1, last = 0;

 

  /**

   * The end index of what matched in the last match operation.

   */

  int oldLast = -1;

 

  /**

   * The index of the last position appended in a substitution.

   */

  int lastAppendPosition = 0;

 

  /**

   * Storage used by nodes to tell what repetition they are on in

   * a pattern, and where groups begin. The nodes themselves are stateless,

   * so they rely on this field to hold state during a match.

   */

  int[] locals;

 

  /**

   * Boolean indicating whether or not more input could change

   * the results of the last match.

   *

   * If hitEnd is true, and a match was found, then more input

   * might cause a different match to be found.

   * If hitEnd is true and a match was not found, then more

   * input could cause a match to be found.

   * If hitEnd is false and a match was found, then more input

   * will not change the match.

   * If hitEnd is false and a match was not found, then more

   * input will not cause a match to be found.

   */

  boolean hitEnd;

 

  /**

   * Boolean indicating whether or not more input could change

   * a positive match into a negative one.

   *

   * If requireEnd is true, and a match was found, then more

   * input could cause the match to be lost.

   * If requireEnd is false and a match was found, then more

   * input might change the match but the match won't be lost.

   * If a match was not found, then requireEnd has no meaning.

   */

  boolean requireEnd;

 

  /**

   * If transparentBounds is true then the boundaries of this

   * matcher's region are transparent to lookahead, lookbehind,

   * and boundary matching constructs that try to see beyond them.

   */

  boolean transparentBounds = false;

 

  /**

   * If anchoringBounds is true then the boundaries of this

   * matcher's region match anchors such as ^ and $.

   */

  boolean anchoringBounds = true;

 

  /**

   * No default constructor.

   */

  Matcher() {

  }

 

/**

 * All matchers have the state used by Pattern during a match.

 */

Matcher(Pattern parent, CharSequence text) {

  this.parentPattern = parent;

  this.text = text;

 

  // Allocate state storage

  int parentGroupCount = Math.max(parent.capturingGroupCount, 10);

  groups = new int[parentGroupCount * 2];

  locals = new int[parent.localCount];

 

  // Put fields into initial states

  reset();

}

....

/**

 * Returns the input subsequence matched by the previous match.

 *

 * <p> For a matcher <i>m</i> with input sequence <i>s</i>,

 * the expressions <i>m.</i><tt>group()</tt> and

 * <i>s.</i><tt>substring(</tt><i>m.</i><tt>start(),</tt> <i>m.</i><tt>end())</tt>

 * are equivalent. </p>

 *

 * <p> Note that some patterns, for example <tt>a*</tt>, match the empty

 * string. This method will return the empty string when the pattern

 * successfully matches the empty string in the input. </p>

 *

 * @return The (possibly empty) subsequence matched by the previous match,

 *     in string form

 *

 * @throws IllegalStateException

 *     If no match has yet been attempted,

 *     or if the previous match operation failed

 */

public String group() {

  return group(0);

}

/**

 * Returns the input subsequence captured by the given group during the

 * previous match operation.

 *

 * <p> For a matcher <i>m</i>, input sequence <i>s</i>, and group index

 * <i>g</i>, the expressions <i>m.</i><tt>group(</tt><i>g</i><tt>)</tt> and

 * <i>s.</i><tt>substring(</tt><i>m.</i><tt>start(</tt><i>g</i><tt>),</tt> <i>m.</i><tt>end(</tt><i>g</i><tt>))</tt>

 * are equivalent. </p>

 *

 * <p> <a href="Pattern.html#cg">Capturing groups</a> are indexed from left

 * to right, starting at one. Group zero denotes the entire pattern, so

 * the expression <tt>m.group(0)</tt> is equivalent to <tt>m.group()</tt>.

 * </p>

 *

 * <p> If the match was successful but the group specified failed to match

 * any part of the input sequence, then <tt>null</tt> is returned. Note

 * that some groups, for example <tt>(a*)</tt>, match the empty string.

 * This method will return the empty string when such a group successfully

 * matches the empty string in the input. </p>

 *

 * @param group

 *     The index of a capturing group in this matcher's pattern

 *

 * @return The (possibly empty) subsequence captured by the group

 *     during the previous match, or <tt>null</tt> if the group

 *     failed to match part of the input

 *

 * @throws IllegalStateException

 *     If no match has yet been attempted,

 *     or if the previous match operation failed

 *

 * @throws IndexOutOfBoundsException

 *     If there is no capturing group in the pattern

 *     with the given index

 */

public String group(int group) {

  if (first < 0)

    throw new IllegalStateException("No match found");

  if (group < 0 || group > groupCount())

    throw new IndexOutOfBoundsException("No group " + group);

  if ((groups[group*2] == -1) || (groups[group*2+1] == -1))

    return null;

  return getSubSequence(groups[group * 2], groups[group * 2 + 1]).toString();

}

/**

 * Attempts to find the next subsequence of the input sequence that matches

 * the pattern.

 *

 * <p> This method starts at the beginning of this matcher's region, or, if

 * a previous invocation of the method was successful and the matcher has

 * not since been reset, at the first character not matched by the previous

 * match.

 *

 * <p> If the match succeeds then more information can be obtained via the

 * <tt>start</tt>, <tt>end</tt>, and <tt>group</tt> methods. </p>

 *

 * @return <tt>true</tt> if, and only if, a subsequence of the input

 *     sequence matches this matcher's pattern

 */

public boolean find() {

  int nextSearchIndex = last;

  if (nextSearchIndex == first)

    nextSearchIndex++;

 

  // If next search starts before region, start it at region

  if (nextSearchIndex < from)

    nextSearchIndex = from;

 

  // If next search starts beyond region then it fails

  if (nextSearchIndex > to) {

    for (int i = 0; i < groups.length; i++)

      groups[i] = -1;

    return false;

  }

  return search(nextSearchIndex);

}

 

/**

 * Initiates a search to find a Pattern within the given bounds.

 * The groups are filled with default values and the match of the root

 * of the state machine is called. The state machine will hold the state

 * of the match as it proceeds in this matcher.

 *

 * Matcher.from is not set here, because it is the "hard" boundary

 * of the start of the search which anchors will set to. The from param

 * is the "soft" boundary of the start of the search, meaning that the

 * regex tries to match at that index but ^ won't match there. Subsequent

 * calls to the search methods start at a new "soft" boundary which is

 * the end of the previous match.

 */

boolean search(int from) {

  this.hitEnd = false;

  this.requireEnd = false;

  from    = from < 0 ? 0 : from;

  this.first = from;

  this.oldLast = oldLast < 0 ? from : oldLast;

  for (int i = 0; i < groups.length; i++)

    groups[i] = -1;

  acceptMode = NOANCHOR;

  boolean result = parentPattern.root.match(this, from, text);

  if (!result)

    this.first = -1;

  this.oldLast = this.last;

  return result;

}

...

} 

boolean search(int from) {

  this.hitEnd = false;

  this.requireEnd = false;

  from    = from < 0 ? 0 : from;

  this.first = from;

  this.oldLast = oldLast < 0 ? from : oldLast;

  for (int i = 0; i < groups.length; i++)

    groups[i] = -1;

  acceptMode = NOANCHOR;

  boolean result = parentPattern.root.match(this, from, text);

  if (!result)

    this.first = -1;

  this.oldLast = this.last;

  return result;

}