Yeah, the LoRA part is from scratch. The LLM backbone in this example is not, this is to provide a concrete example. But you could apply the exact same LoRA from scratch code to a pure PyTorch model if you wanted to:

E.g.

    class MultilayerPerceptron(nn.Module):

        def __init__(self, num_features, num_hidden_1, num_hidden_2, num_classes):
            super().__init__()

            self.layers = nn.Sequential(
                nn.Linear(num_features, num_hidden_1),
                nn.ReLU(),
                nn.Linear(num_hidden_1, num_hidden_2),
                nn.ReLU(),
                nn.Linear(num_hidden_2, num_classes)
            )

        def forward(self, x):
            x = self.layers(x)
            return x

    model = MultilayerPerceptron(
        num_features=num_features,
        num_hidden_1=num_hidden_1,
        num_hidden_2=num_hidden_2, 
        num_classes=num_classes
    )

    model.layers[0] = LinearWithLoRA(model.layers[0], rank=4, alpha=1)
    model.layers[2] = LinearWithLoRA(model.layers[2], rank=4, alpha=1)
    model.layers[4] = LinearWithLoRA(model.layers[4], rank=4, alpha=1)