Added support for multiple tags on a single model
This commit is contained in:
jeffser
@@ -0,0 +1,230 @@
|
||||
"""
|
||||
This module contains provisional support for SOCKS proxies from within
|
||||
urllib3. This module supports SOCKS4, SOCKS4A (an extension of SOCKS4), and
|
||||
SOCKS5. To enable its functionality, either install PySocks or install this
|
||||
module with the ``socks`` extra.
|
||||
|
||||
The SOCKS implementation supports the full range of urllib3 features. It also
|
||||
supports the following SOCKS features:
|
||||
|
||||
- SOCKS4A (``proxy_url='socks4a://...``)
|
||||
- SOCKS4 (``proxy_url='socks4://...``)
|
||||
- SOCKS5 with remote DNS (``proxy_url='socks5h://...``)
|
||||
- SOCKS5 with local DNS (``proxy_url='socks5://...``)
|
||||
- Usernames and passwords for the SOCKS proxy
|
||||
|
||||
.. note::
|
||||
It is recommended to use ``socks5h://`` or ``socks4a://`` schemes in
|
||||
your ``proxy_url`` to ensure that DNS resolution is done from the remote
|
||||
server instead of client-side when connecting to a domain name.
|
||||
|
||||
SOCKS4 supports IPv4 and domain names with the SOCKS4A extension. SOCKS5
|
||||
supports IPv4, IPv6, and domain names.
|
||||
|
||||
When connecting to a SOCKS4 proxy the ``username`` portion of the ``proxy_url``
|
||||
will be sent as the ``userid`` section of the SOCKS request:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
proxy_url="socks4a://<userid>@proxy-host"
|
||||
|
||||
When connecting to a SOCKS5 proxy the ``username`` and ``password`` portion
|
||||
of the ``proxy_url`` will be sent as the username/password to authenticate
|
||||
with the proxy:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
proxy_url="socks5h://<username>:<password>@proxy-host"
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
try:
|
||||
import socks # type: ignore[import-not-found]
|
||||
except ImportError:
|
||||
import warnings
|
||||
|
||||
from ..exceptions import DependencyWarning
|
||||
|
||||
warnings.warn(
|
||||
(
|
||||
"SOCKS support in urllib3 requires the installation of optional "
|
||||
"dependencies: specifically, PySocks. For more information, see "
|
||||
"https://urllib3.readthedocs.io/en/latest/advanced-usage.html#socks-proxies"
|
||||
),
|
||||
DependencyWarning,
|
||||
)
|
||||
raise
|
||||
|
||||
import typing
|
||||
from socket import timeout as SocketTimeout
|
||||
|
||||
from ..connection import HTTPConnection, HTTPSConnection
|
||||
from ..connectionpool import HTTPConnectionPool, HTTPSConnectionPool
|
||||
from ..exceptions import ConnectTimeoutError, NewConnectionError
|
||||
from ..poolmanager import PoolManager
|
||||
from ..util.url import parse_url
|
||||
|
||||
try:
|
||||
import ssl
|
||||
except ImportError:
|
||||
ssl = None # type: ignore[assignment]
|
||||
|
||||
from typing import TypedDict
|
||||
|
||||
|
||||
class _TYPE_SOCKS_OPTIONS(TypedDict):
|
||||
socks_version: int
|
||||
proxy_host: str | None
|
||||
proxy_port: str | None
|
||||
username: str | None
|
||||
password: str | None
|
||||
rdns: bool
|
||||
|
||||
|
||||
class SOCKSConnection(HTTPConnection):
|
||||
"""
|
||||
A plain-text HTTP connection that connects via a SOCKS proxy.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
_socks_options: _TYPE_SOCKS_OPTIONS,
|
||||
*args: typing.Any,
|
||||
**kwargs: typing.Any,
|
||||
) -> None:
|
||||
self._socks_options = _socks_options
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
def _new_conn(self) -> socks.socksocket:
|
||||
"""
|
||||
Establish a new connection via the SOCKS proxy.
|
||||
"""
|
||||
extra_kw: dict[str, typing.Any] = {}
|
||||
if self.source_address:
|
||||
extra_kw["source_address"] = self.source_address
|
||||
|
||||
if self.socket_options:
|
||||
extra_kw["socket_options"] = self.socket_options
|
||||
|
||||
try:
|
||||
conn = socks.create_connection(
|
||||
(self.host, self.port),
|
||||
proxy_type=self._socks_options["socks_version"],
|
||||
proxy_addr=self._socks_options["proxy_host"],
|
||||
proxy_port=self._socks_options["proxy_port"],
|
||||
proxy_username=self._socks_options["username"],
|
||||
proxy_password=self._socks_options["password"],
|
||||
proxy_rdns=self._socks_options["rdns"],
|
||||
timeout=self.timeout,
|
||||
**extra_kw,
|
||||
)
|
||||
|
||||
except SocketTimeout as e:
|
||||
raise ConnectTimeoutError(
|
||||
self,
|
||||
f"Connection to {self.host} timed out. (connect timeout={self.timeout})",
|
||||
) from e
|
||||
|
||||
except socks.ProxyError as e:
|
||||
# This is fragile as hell, but it seems to be the only way to raise
|
||||
# useful errors here.
|
||||
if e.socket_err:
|
||||
error = e.socket_err
|
||||
if isinstance(error, SocketTimeout):
|
||||
raise ConnectTimeoutError(
|
||||
self,
|
||||
f"Connection to {self.host} timed out. (connect timeout={self.timeout})",
|
||||
) from e
|
||||
else:
|
||||
# Adding `from e` messes with coverage somehow, so it's omitted.
|
||||
# See #2386.
|
||||
raise NewConnectionError(
|
||||
self, f"Failed to establish a new connection: {error}"
|
||||
)
|
||||
else:
|
||||
raise NewConnectionError(
|
||||
self, f"Failed to establish a new connection: {e}"
|
||||
) from e
|
||||
|
||||
except OSError as e: # Defensive: PySocks should catch all these.
|
||||
raise NewConnectionError(
|
||||
self, f"Failed to establish a new connection: {e}"
|
||||
) from e
|
||||
|
||||
return conn
|
||||
|
||||
|
||||
# We don't need to duplicate the Verified/Unverified distinction from
|
||||
# urllib3/connection.py here because the HTTPSConnection will already have been
|
||||
# correctly set to either the Verified or Unverified form by that module. This
|
||||
# means the SOCKSHTTPSConnection will automatically be the correct type.
|
||||
class SOCKSHTTPSConnection(SOCKSConnection, HTTPSConnection):
|
||||
pass
|
||||
|
||||
|
||||
class SOCKSHTTPConnectionPool(HTTPConnectionPool):
|
||||
ConnectionCls = SOCKSConnection
|
||||
|
||||
|
||||
class SOCKSHTTPSConnectionPool(HTTPSConnectionPool):
|
||||
ConnectionCls = SOCKSHTTPSConnection
|
||||
|
||||
|
||||
class SOCKSProxyManager(PoolManager):
|
||||
"""
|
||||
A version of the urllib3 ProxyManager that routes connections via the
|
||||
defined SOCKS proxy.
|
||||
"""
|
||||
|
||||
pool_classes_by_scheme = {
|
||||
"http": SOCKSHTTPConnectionPool,
|
||||
"https": SOCKSHTTPSConnectionPool,
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
proxy_url: str,
|
||||
username: str | None = None,
|
||||
password: str | None = None,
|
||||
num_pools: int = 10,
|
||||
headers: typing.Mapping[str, str] | None = None,
|
||||
**connection_pool_kw: typing.Any,
|
||||
):
|
||||
parsed = parse_url(proxy_url)
|
||||
|
||||
if username is None and password is None and parsed.auth is not None:
|
||||
split = parsed.auth.split(":")
|
||||
if len(split) == 2:
|
||||
username, password = split
|
||||
if parsed.scheme == "socks5":
|
||||
socks_version = socks.PROXY_TYPE_SOCKS5
|
||||
rdns = False
|
||||
elif parsed.scheme == "socks5h":
|
||||
socks_version = socks.PROXY_TYPE_SOCKS5
|
||||
rdns = True
|
||||
elif parsed.scheme == "socks4":
|
||||
socks_version = socks.PROXY_TYPE_SOCKS4
|
||||
rdns = False
|
||||
elif parsed.scheme == "socks4a":
|
||||
socks_version = socks.PROXY_TYPE_SOCKS4
|
||||
rdns = True
|
||||
else:
|
||||
raise ValueError(f"Unable to determine SOCKS version from {proxy_url}")
|
||||
|
||||
self.proxy_url = proxy_url
|
||||
|
||||
socks_options = {
|
||||
"socks_version": socks_version,
|
||||
"proxy_host": parsed.host,
|
||||
"proxy_port": parsed.port,
|
||||
"username": username,
|
||||
"password": password,
|
||||
"rdns": rdns,
|
||||
}
|
||||
connection_pool_kw["_socks_options"] = socks_options
|
||||
|
||||
super().__init__(num_pools, headers, **connection_pool_kw)
|
||||
|
||||
self.pool_classes_by_scheme = SOCKSProxyManager.pool_classes_by_scheme
|
||||
@@ -0,0 +1,263 @@
|
||||
#
|
||||
# The Python Imaging Library.
|
||||
# $Id$
|
||||
#
|
||||
# image palette object
|
||||
#
|
||||
# History:
|
||||
# 1996-03-11 fl Rewritten.
|
||||
# 1997-01-03 fl Up and running.
|
||||
# 1997-08-23 fl Added load hack
|
||||
# 2001-04-16 fl Fixed randint shadow bug in random()
|
||||
#
|
||||
# Copyright (c) 1997-2001 by Secret Labs AB
|
||||
# Copyright (c) 1996-1997 by Fredrik Lundh
|
||||
#
|
||||
# See the README file for information on usage and redistribution.
|
||||
#
|
||||
from __future__ import annotations
|
||||
|
||||
import array
|
||||
from typing import Sequence
|
||||
|
||||
from . import GimpGradientFile, GimpPaletteFile, ImageColor, PaletteFile
|
||||
|
||||
|
||||
class ImagePalette:
|
||||
"""
|
||||
Color palette for palette mapped images
|
||||
|
||||
:param mode: The mode to use for the palette. See:
|
||||
:ref:`concept-modes`. Defaults to "RGB"
|
||||
:param palette: An optional palette. If given, it must be a bytearray,
|
||||
an array or a list of ints between 0-255. The list must consist of
|
||||
all channels for one color followed by the next color (e.g. RGBRGBRGB).
|
||||
Defaults to an empty palette.
|
||||
"""
|
||||
|
||||
def __init__(self, mode: str = "RGB", palette: Sequence[int] | None = None) -> None:
|
||||
self.mode = mode
|
||||
self.rawmode = None # if set, palette contains raw data
|
||||
self.palette = palette or bytearray()
|
||||
self.dirty: int | None = None
|
||||
|
||||
@property
|
||||
def palette(self):
|
||||
return self._palette
|
||||
|
||||
@palette.setter
|
||||
def palette(self, palette):
|
||||
self._colors = None
|
||||
self._palette = palette
|
||||
|
||||
@property
|
||||
def colors(self):
|
||||
if self._colors is None:
|
||||
mode_len = len(self.mode)
|
||||
self._colors = {}
|
||||
for i in range(0, len(self.palette), mode_len):
|
||||
color = tuple(self.palette[i : i + mode_len])
|
||||
if color in self._colors:
|
||||
continue
|
||||
self._colors[color] = i // mode_len
|
||||
return self._colors
|
||||
|
||||
@colors.setter
|
||||
def colors(self, colors):
|
||||
self._colors = colors
|
||||
|
||||
def copy(self):
|
||||
new = ImagePalette()
|
||||
|
||||
new.mode = self.mode
|
||||
new.rawmode = self.rawmode
|
||||
if self.palette is not None:
|
||||
new.palette = self.palette[:]
|
||||
new.dirty = self.dirty
|
||||
|
||||
return new
|
||||
|
||||
def getdata(self):
|
||||
"""
|
||||
Get palette contents in format suitable for the low-level
|
||||
``im.putpalette`` primitive.
|
||||
|
||||
.. warning:: This method is experimental.
|
||||
"""
|
||||
if self.rawmode:
|
||||
return self.rawmode, self.palette
|
||||
return self.mode, self.tobytes()
|
||||
|
||||
def tobytes(self):
|
||||
"""Convert palette to bytes.
|
||||
|
||||
.. warning:: This method is experimental.
|
||||
"""
|
||||
if self.rawmode:
|
||||
msg = "palette contains raw palette data"
|
||||
raise ValueError(msg)
|
||||
if isinstance(self.palette, bytes):
|
||||
return self.palette
|
||||
arr = array.array("B", self.palette)
|
||||
return arr.tobytes()
|
||||
|
||||
# Declare tostring as an alias for tobytes
|
||||
tostring = tobytes
|
||||
|
||||
def _new_color_index(self, image=None, e=None):
|
||||
if not isinstance(self.palette, bytearray):
|
||||
self._palette = bytearray(self.palette)
|
||||
index = len(self.palette) // 3
|
||||
special_colors = ()
|
||||
if image:
|
||||
special_colors = (
|
||||
image.info.get("background"),
|
||||
image.info.get("transparency"),
|
||||
)
|
||||
while index in special_colors:
|
||||
index += 1
|
||||
if index >= 256:
|
||||
if image:
|
||||
# Search for an unused index
|
||||
for i, count in reversed(list(enumerate(image.histogram()))):
|
||||
if count == 0 and i not in special_colors:
|
||||
index = i
|
||||
break
|
||||
if index >= 256:
|
||||
msg = "cannot allocate more than 256 colors"
|
||||
raise ValueError(msg) from e
|
||||
return index
|
||||
|
||||
def getcolor(self, color, image=None) -> int:
|
||||
"""Given an rgb tuple, allocate palette entry.
|
||||
|
||||
.. warning:: This method is experimental.
|
||||
"""
|
||||
if self.rawmode:
|
||||
msg = "palette contains raw palette data"
|
||||
raise ValueError(msg)
|
||||
if isinstance(color, tuple):
|
||||
if self.mode == "RGB":
|
||||
if len(color) == 4:
|
||||
if color[3] != 255:
|
||||
msg = "cannot add non-opaque RGBA color to RGB palette"
|
||||
raise ValueError(msg)
|
||||
color = color[:3]
|
||||
elif self.mode == "RGBA":
|
||||
if len(color) == 3:
|
||||
color += (255,)
|
||||
try:
|
||||
return self.colors[color]
|
||||
except KeyError as e:
|
||||
# allocate new color slot
|
||||
index = self._new_color_index(image, e)
|
||||
self.colors[color] = index
|
||||
if index * 3 < len(self.palette):
|
||||
self._palette = (
|
||||
self.palette[: index * 3]
|
||||
+ bytes(color)
|
||||
+ self.palette[index * 3 + 3 :]
|
||||
)
|
||||
else:
|
||||
self._palette += bytes(color)
|
||||
self.dirty = 1
|
||||
return index
|
||||
else:
|
||||
msg = f"unknown color specifier: {repr(color)}"
|
||||
raise ValueError(msg)
|
||||
|
||||
def save(self, fp):
|
||||
"""Save palette to text file.
|
||||
|
||||
.. warning:: This method is experimental.
|
||||
"""
|
||||
if self.rawmode:
|
||||
msg = "palette contains raw palette data"
|
||||
raise ValueError(msg)
|
||||
if isinstance(fp, str):
|
||||
fp = open(fp, "w")
|
||||
fp.write("# Palette\n")
|
||||
fp.write(f"# Mode: {self.mode}\n")
|
||||
for i in range(256):
|
||||
fp.write(f"{i}")
|
||||
for j in range(i * len(self.mode), (i + 1) * len(self.mode)):
|
||||
try:
|
||||
fp.write(f" {self.palette[j]}")
|
||||
except IndexError:
|
||||
fp.write(" 0")
|
||||
fp.write("\n")
|
||||
fp.close()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------
|
||||
# Internal
|
||||
|
||||
|
||||
def raw(rawmode, data) -> ImagePalette:
|
||||
palette = ImagePalette()
|
||||
palette.rawmode = rawmode
|
||||
palette.palette = data
|
||||
palette.dirty = 1
|
||||
return palette
|
||||
|
||||
|
||||
# --------------------------------------------------------------------
|
||||
# Factories
|
||||
|
||||
|
||||
def make_linear_lut(black, white):
|
||||
if black == 0:
|
||||
return [white * i // 255 for i in range(256)]
|
||||
|
||||
msg = "unavailable when black is non-zero"
|
||||
raise NotImplementedError(msg) # FIXME
|
||||
|
||||
|
||||
def make_gamma_lut(exp):
|
||||
return [int(((i / 255.0) ** exp) * 255.0 + 0.5) for i in range(256)]
|
||||
|
||||
|
||||
def negative(mode="RGB"):
|
||||
palette = list(range(256 * len(mode)))
|
||||
palette.reverse()
|
||||
return ImagePalette(mode, [i // len(mode) for i in palette])
|
||||
|
||||
|
||||
def random(mode="RGB"):
|
||||
from random import randint
|
||||
|
||||
palette = [randint(0, 255) for _ in range(256 * len(mode))]
|
||||
return ImagePalette(mode, palette)
|
||||
|
||||
|
||||
def sepia(white="#fff0c0"):
|
||||
bands = [make_linear_lut(0, band) for band in ImageColor.getrgb(white)]
|
||||
return ImagePalette("RGB", [bands[i % 3][i // 3] for i in range(256 * 3)])
|
||||
|
||||
|
||||
def wedge(mode="RGB"):
|
||||
palette = list(range(256 * len(mode)))
|
||||
return ImagePalette(mode, [i // len(mode) for i in palette])
|
||||
|
||||
|
||||
def load(filename):
|
||||
# FIXME: supports GIMP gradients only
|
||||
|
||||
with open(filename, "rb") as fp:
|
||||
for paletteHandler in [
|
||||
GimpPaletteFile.GimpPaletteFile,
|
||||
GimpGradientFile.GimpGradientFile,
|
||||
PaletteFile.PaletteFile,
|
||||
]:
|
||||
try:
|
||||
fp.seek(0)
|
||||
lut = paletteHandler(fp).getpalette()
|
||||
if lut:
|
||||
break
|
||||
except (SyntaxError, ValueError):
|
||||
pass
|
||||
else:
|
||||
msg = "cannot load palette"
|
||||
raise OSError(msg)
|
||||
|
||||
return lut # data, rawmode
|
||||
Reference in New Issue
Block a user