Lines Matching full:line
58 lines = [line.strip() for line in f]
79 We allow that a user-agent: line is not preceded by
83 # 1: saw user-agent line
84 # 2: saw an allow or disallow line
89 for line in lines:
91 if not line:
99 # remove optional comment and strip line
100 i = line.find('#')
102 line = line[:i]
103 line = line.strip()
104 if not line:
106 line = line.split(':', 1)
107 if len(line) == 2:
108 line[0] = line[0].strip().lower()
109 line[1] = urllib.unquote(line[1].strip())
110 if line[0] == "user-agent":
114 entry.useragents.append(line[1])
116 elif line[0] == "disallow":
118 entry.rulelines.append(RuleLine(line[1], False))
120 elif line[0] == "allow":
122 entry.rulelines.append(RuleLine(line[1], True))
157 """A rule line is a single "Allow:" (allowance==True) or "Disallow:"
183 for line in self.rulelines:
184 ret.extend([str(line), "\n"])
204 for line in self.rulelines:
205 if line.applies_to(filename):
206 return line.allowance
