From e9249b05f4701b8b52f051e98859178a38133ea9 Mon Sep 17 00:00:00 2001 From: GRMrGecko Date: Mon, 14 Jul 2025 10:22:59 -0500 Subject: [PATCH] First commit --- .gitignore | 1 + LICENSE.txt | 19 ++++ README.md | 20 +++++ config.yaml | 4 + go.mod | 9 ++ go.sum | 7 ++ main.go | 252 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 312 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE.txt create mode 100644 README.md create mode 100644 config.yaml create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b91883b --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +pip-proxy diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..c11d2f8 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,19 @@ +Copyright (c) 2025 Mr. Gecko's Media (James Coleman). http://mrgeckosmedia.com/ + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..6f55d24 --- /dev/null +++ b/README.md @@ -0,0 +1,20 @@ +# pip-proxy + +A proxy server to set as your PIP mirror and limit what versions of a package is available, based on python version and or individual package versions. + +## Using the proxy as an mirror + +In either `/etc/pip.conf` or `~/pip/pip.conf`: +```ini +[global] +index-url=http://192.168.122.1:8080/simple +trusted-host=192.168.122.1 +``` + +In `/root/.pydistutils.cfg`: +```ini +[easy_install] +index-url=http://192.168.122.1:8080/simple +``` + +To configure the constraints, edit config.yaml diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..d8587e7 --- /dev/null +++ b/config.yaml @@ -0,0 +1,4 @@ +--- +max_python_version: 3.6 +package_version_limits: + xmlsec: 1.3.3 diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..2cba0ab --- /dev/null +++ b/go.mod @@ -0,0 +1,9 @@ +module github.com/grmrgecko/pip-proxy + +go 1.24.4 + +require github.com/hashicorp/go-version v1.7.0 + +require github.com/gobwas/glob v0.2.3 + +require gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..4f876d3 --- /dev/null +++ b/go.sum @@ -0,0 +1,7 @@ +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= +github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go new file mode 100644 index 0000000..589f74e --- /dev/null +++ b/main.go @@ -0,0 +1,252 @@ +package main + +import ( + "bufio" + "bytes" + "compress/gzip" + "fmt" + "html" + "io" + "log" + "net/http" + "os" + "regexp" + "strconv" + "strings" + + "github.com/gobwas/glob" + "github.com/hashicorp/go-version" + "gopkg.in/yaml.v3" +) + +// Config file structure. +type Config struct { + HttpPort uint `yaml:"hddp_port"` + HttpBind string `yaml:"http_bind"` + MaxPythonVersion string `yaml:"max_python_version"` + PackageVersionLimits map[string]string `yaml:"package_version_limits"` +} + +var config *Config + +// Handle http proxy request. +func handleRequest(rw http.ResponseWriter, req *http.Request) { + // Replace the scheme and host to pypi.org. + url := req.URL + url.Scheme = "https" + url.Host = "pypi.org" + + // Create a new request based on the original + outreq, err := http.NewRequest(req.Method, url.String(), req.Body) + if err != nil { + http.Error(rw, err.Error(), http.StatusInternalServerError) + return + } + + // Copy headers from original request, and correct the host header. + outreq.Header = req.Header + outreq.Header.Set("Host", url.Host) + + // Make sure we close the request body after the function call is done. + if outreq.Body != nil { + defer outreq.Body.Close() + } + + // Send the request. + res, err := http.DefaultTransport.RoundTrip(outreq) + if err != nil { + http.Error(rw, err.Error(), http.StatusServiceUnavailable) + return + } + defer res.Body.Close() + + // Copy headers received to requester. + for k, vv := range res.Header { + for _, v := range vv { + rw.Header().Add(k, v) + } + } + + // Log the request. + log.Println(req.Method, url, res.StatusCode) + + // Verify this is `/simple/package/` request. + pathS := strings.Split(req.URL.Path, "/") + if len(pathS) == 4 { + // Get the package being requested, and check if there is a version constraint on it. + pkg := pathS[2] + pkgVerMax, hasVerConstraint := config.PackageVersionLimits[pkg] + var pkgVer *version.Version + if hasVerConstraint { + pkgVer, _ = version.NewVersion(pkgVerMax) + } + + // Buffer to store the modified response. + bodyBuff := new(bytes.Buffer) + + // Determine body, gzip encoded or plain text. + var bodyReader io.Reader = res.Body + gzipEncoded := res.Header.Get("Content-Encoding") == "gzip" + if gzipEncoded { + bodyReader, err = gzip.NewReader(res.Body) + } + + // Setup scanner and matching variables. + scanner := bufio.NewScanner(bodyReader) + constraintsRx := regexp.MustCompile(`data-requires-python="([^"]+)"`) + pkgVersionRx := regexp.MustCompile(`]+>[^<]+-([0-9]+.[0-9]+.[0-9]+)[-.][^<]+`) + pyVersion, _ := version.NewVersion(config.MaxPythonVersion) + + // Scan each line, and apply version constraints. + for scanner.Scan() { + line := scanner.Text() + + // Check if this has the python version constraints. + matches := constraintsRx.FindAllStringSubmatch(line, 1) + if len(matches) == 1 { + // Compare and skip if the constraints are not matched. + rules := strings.Split(html.UnescapeString(matches[0][1]), ",") + versionMatch := true + for _, rule := range rules { + var cmpB strings.Builder + var verB strings.Builder + wildCard := false + for _, b := range rule { + if b == '<' || b == '>' || b == '=' || b == '!' || b == '~' { + cmpB.WriteRune(b) + } else if b == ' ' { + continue + } else { + if b == '*' { + wildCard = true + } + verB.WriteRune(b) + } + } + var g glob.Glob + var ver *version.Version + cmp := cmpB.String() + if wildCard && cmp == "==" || cmp == "!=" { + g = glob.MustCompile(verB.String()) + } else { + verS := verB.String() + if wildCard { + verS = strings.ReplaceAll(verS, "*", "0") + } + ver, err = version.NewVersion(verS) + if err != nil { + log.Println(err) + } + } + switch cmp { + case "==": + if wildCard { + if !g.Match(config.MaxPythonVersion) { + versionMatch = false + } + } else if !pyVersion.Equal(ver) { + versionMatch = false + } + case "!=": + if wildCard { + if g.Match(config.MaxPythonVersion) { + versionMatch = false + } + } else if pyVersion.Equal(ver) { + versionMatch = false + } + case "=~": + g = glob.MustCompile(fmt.Sprintf("%s.*", config.MaxPythonVersion)) + if !g.Match(ver.String()) { + versionMatch = false + } + case ">=": + if !pyVersion.GreaterThanOrEqual(ver) { + versionMatch = false + } + case ">": + if !pyVersion.GreaterThan(ver) { + versionMatch = false + } + case "<=": + if !pyVersion.LessThanOrEqual(ver) { + versionMatch = false + } + case "<": + if !pyVersion.LessThan(ver) { + versionMatch = false + } + } + } + + if !versionMatch { + continue + } + } + + // If there is a package version constraint, compare the package version and skip if not met. + if hasVerConstraint { + matches = pkgVersionRx.FindAllStringSubmatch(line, 1) + if len(matches) == 1 { + ver, err := version.NewVersion(matches[0][1]) + if err != nil { + log.Println(err) + } else { + if pkgVer.LessThan(ver) { + continue + } + } + } + } + + // Write the read line. + fmt.Fprintln(bodyBuff, line) + } + + // Get the modified data and compress with gzip if needed. + bodyBytes := bodyBuff.Bytes() + if gzipEncoded { + bodyBuff = new(bytes.Buffer) + gzipWriter := gzip.NewWriter(bodyBuff) + gzipWriter.Write(bodyBytes) + gzipWriter.Close() + bodyBytes = bodyBuff.Bytes() + } + + // Update the content length. + rw.Header().Set("Content-Length", strconv.Itoa(len(bodyBytes))) + + // Send headers. + rw.WriteHeader(res.StatusCode) + + // Send the modified body. + rw.Write(bodyBytes) + } else { + // Just copy the body as there is nothing to limit here. + rw.WriteHeader(res.StatusCode) + io.Copy(rw, res.Body) + } +} + +func main() { + // Read the yaml configuration. + yamlD, err := os.ReadFile("config.yaml") + if err != nil { + log.Fatal(err) + } + config = &Config{ + HttpPort: 8080, + } + err = yaml.Unmarshal(yamlD, config) + if err != nil { + log.Fatal(err) + } + + // Start the server. + bindAddr := fmt.Sprintf("%s:%d", config.HttpBind, config.HttpPort) + log.Println("Starting proxy server on", bindAddr) + err = http.ListenAndServe(bindAddr, http.HandlerFunc(handleRequest)) + if err != nil { + log.Fatal(err) + } +}